def _load_config(self): """ Loads the configuration from YAML, if no override config was passed in initially. """ if ( self.config ): # any config being pre-set at init will short circuit out, but not a plain {} return # Verify that we're in a project repo_root = self.repo_root if not repo_root: raise NotInProject( "No git repository was found in the current path. You must be in a git repository to set up and use CCI for a project." ) # Verify that the project's root has a config file if not self.config_project_path: raise ProjectConfigNotFound( "The file {} was not found in the repo root: {}. Are you in a CumulusCI Project directory?".format( self.config_filename, repo_root ) ) # Load the project's yaml config file with open(self.config_project_path, "r") as f_config: project_config = ordered_yaml_load(f_config) if project_config: self.config_project.update(project_config) # Load the local project yaml config file if it exists if self.config_project_local_path: with open(self.config_project_local_path, "r") as f_local_config: local_config = ordered_yaml_load(f_local_config) if local_config: self.config_project_local.update(local_config) # merge in any additional yaml that was passed along if self.additional_yaml: additional_yaml_config = ordered_yaml_load(self.additional_yaml) if additional_yaml_config: self.config_additional_yaml.update(additional_yaml_config) self.config = merge_config( OrderedDict( [ ("global_config", self.config_global), ("global_local", self.config_global_local), ("project_config", self.config_project), ("project_local_config", self.config_project_local), ("additional_yaml", self.config_additional_yaml), ] ) )
def _load_config(self): """ Loads the configuration from YAML, if no override config was passed in initially. """ if ( self.config ): # any config being pre-set at init will short circuit out, but not a plain {} return # Verify that we're in a project repo_root = self.repo_root if not repo_root: raise NotInProject( "No git repository was found in the current path. You must be in a git repository to set up and use CCI for a project." ) # Verify that the project's root has a config file if not self.config_project_path: raise ProjectConfigNotFound( "The file {} was not found in the repo root: {}. Are you in a CumulusCI Project directory?" .format(self.config_filename, repo_root)) # Load the project's yaml config file with open(self.config_project_path, "r") as f_config: project_config = ordered_yaml_load(f_config) if project_config: self.config_project.update(project_config) # Load the local project yaml config file if it exists if self.config_project_local_path: with open(self.config_project_local_path, "r") as f_local_config: local_config = ordered_yaml_load(f_local_config) if local_config: self.config_project_local.update(local_config) # merge in any additional yaml that was passed along if self.additional_yaml: additional_yaml_config = ordered_yaml_load(self.additional_yaml) if additional_yaml_config: self.config_additional_yaml.update(additional_yaml_config) self.config = merge_config({ "global_config": self.config_global, "global_local": self.config_global_local, "project_config": self.config_project, "project_local_config": self.config_project_local, "additional_yaml": self.config_additional_yaml, })
def test_load_global_config_no_local(self, mock_class): mock_class.return_value = self.tempdir_home config = BaseGlobalConfig() with open(__location__ + "/../../cumulusci.yml", "r") as f_expected_config: expected_config = ordered_yaml_load(f_expected_config) self.assertEqual(config.config, expected_config)
def test_run_task(self): t = _make_task(bulkdata.GenerateMapping, {"options": { "path": "mapping.yaml" }}) t.project_config.project__package__api_version = "45.0" describe_data = { "Account": { "fields": [self._mock_field("Id"), self._mock_field("Custom__c")] }, "Child__c": { "fields": [ self._mock_field("Id"), self._mock_field( "Account__c", field_type="reference", referenceTo=["Account"], relationshipOrder=None, ), ] }, } self._prepare_describe_mock(t, describe_data) with temporary_dir(): t() with open("mapping.yaml", "r") as fh: content = ordered_yaml_load(fh) self.assertEqual(["Insert Account", "Insert Child__c"], list(content.keys())) self.assertEqual("Account", t.mapping["Insert Account"]["sf_object"]) self.assertEqual("account", t.mapping["Insert Account"]["table"]) self.assertEqual(["Id", "Custom__c"], list( t.mapping["Insert Account"]["fields"].keys())) self.assertEqual("sf_id", t.mapping["Insert Account"]["fields"]["Id"]) self.assertEqual( "custom__c", t.mapping["Insert Account"]["fields"]["Custom__c"]) self.assertEqual("Child__c", t.mapping["Insert Child__c"]["sf_object"]) self.assertEqual("child__c", t.mapping["Insert Child__c"]["table"]) self.assertEqual( ["Id"], list(t.mapping["Insert Child__c"]["fields"].keys())) self.assertEqual( ["Account__c"], list(t.mapping["Insert Child__c"]["lookups"].keys())) self.assertEqual("sf_id", t.mapping["Insert Child__c"]["fields"]["Id"]) self.assertEqual( "account", t.mapping["Insert Child__c"]["lookups"]["Account__c"]["table"], )
def test_load_global_config_empty_local(self, mock_class): self._create_global_config_local("") mock_class.return_value = self.tempdir_home config = BaseGlobalConfig() with open(__location__ + "/../../cumulusci.yml", "r") as f_expected_config: expected_config = ordered_yaml_load(f_expected_config) self.assertEqual(config.config, expected_config)
def _generate_data(self, db_url, mapping_file_path): """Generate all of the data""" with open(mapping_file_path, "r") as f: mappings = ordered_yaml_load(f) session, base = init_db(db_url, mappings) self.generate_data(session, base) self.session.commit()
def _load_config(self): """ Loads the local configuration """ # load the global config with open(self.config_global_path, "r") as f_config: config = ordered_yaml_load(f_config) self.config_global = config # Load the local config if self.config_global_local_path: config = ordered_yaml_load(open(self.config_global_local_path, "r")) self.config_global_local = config self.config = merge_config({ "global_config": self.config_global, "global_local": self.config_global_local, })
def _generate_data(self, db_url, mapping_file_path, num_records, current_batch_num): """Generate all of the data""" with open(mapping_file_path, "r") as f: mappings = ordered_yaml_load(f) session, engine, base = self.init_db(db_url, mappings) self.generate_data(session, engine, base, num_records, current_batch_num) session.commit()
def test_load_global_config_with_local(self, mock_class): local_yaml = "tasks:\n newtesttask:\n description: test description" self._create_global_config_local(local_yaml) mock_class.return_value = self.tempdir_home config = BaseGlobalConfig() with open(__location__ + "/../../cumulusci.yml", "r") as f_expected_config: expected_config = ordered_yaml_load(f_expected_config) expected_config["tasks"]["newtesttask"] = {} expected_config["tasks"]["newtesttask"]["description"] = "test description" self.assertEqual(config.config, expected_config)
def _load_config(self): """ Loads the local configuration """ # load the global config with open(self.config_global_path, "r") as f_config: config = ordered_yaml_load(f_config) self.config_global = config # Load the local config if self.config_global_local_path: config = ordered_yaml_load(open(self.config_global_local_path, "r")) self.config_global_local = config self.config = merge_config( OrderedDict( [ ("global_config", self.config_global), ("global_local", self.config_global_local), ] ) )
def test_create_table_modern_id_mapping(self): mapping_file = os.path.join(os.path.dirname(__file__), "mapping_v2.yml") with open(mapping_file, "r") as fh: content = ordered_yaml_load(fh) account_mapping = content["Insert Contacts"] with temporary_dir() as d: tmp_db_path = os.path.join(d, "temp.db") engine, metadata = create_db_file(tmp_db_path) t = create_table(account_mapping, metadata) assert t.name == "contacts" assert isinstance(t.columns["id"].type, Integer) assert isinstance(t.columns["first_name"].type, Unicode) assert isinstance(t.columns["last_name"].type, Unicode) assert isinstance(t.columns["email"].type, Unicode)
def __init__( self, directory, api_version, package_name=None, managed=None, delete=None, install_class=None, uninstall_class=None, ): with open(__location__ + "/metadata_map.yml", "r") as f_metadata_map: self.metadata_map = ordered_yaml_load(f_metadata_map) self.directory = directory self.api_version = api_version self.package_name = package_name self.managed = managed self.delete = delete self.install_class = install_class self.uninstall_class = uninstall_class self.types = []
def __init__( self, directory, api_version, package_name=None, managed=None, delete=None, install_class=None, uninstall_class=None, types=None, ): with open(__location__ + "/metadata_map.yml", "r") as f_metadata_map: self.metadata_map = ordered_yaml_load(f_metadata_map) self.directory = directory self.api_version = api_version self.package_name = package_name self.managed = managed self.delete = delete self.install_class = install_class self.uninstall_class = uninstall_class self.types = types or []
def _init_mapping(self): with open(self.options["mapping"], "r") as f: self.mappings = ordered_yaml_load(f)
def get_base_config(): path = os.path.abspath(os.path.join(__location__, "cumulusci.yml")) with open(path, "r") as f: return ordered_yaml_load(f)
def process_github_dependency(self, dependency, indent=None, include_beta=None): if not indent: indent = "" self.logger.info( "{}Processing dependencies from Github repo {}".format( indent, dependency["github"])) skip = dependency.get("skip") if not isinstance(skip, list): skip = [skip] # Initialize github3.py API against repo gh = self.get_github_api() repo_owner, repo_name = dependency["github"].split("/")[3:5] if repo_name.endswith(".git"): repo_name = repo_name[:-4] repo = gh.repository(repo_owner, repo_name) # Determine the ref if specified kwargs = {} if "tag" in dependency: tag = dependency["tag"] kwargs["ref"] = tag else: tag = None # Get the cumulusci.yml file contents = repo.file_contents("cumulusci.yml", **kwargs) cumulusci_yml = ordered_yaml_load(contents.decoded) # Get the namespace from the cumulusci.yml if set namespace = cumulusci_yml.get("project", {}).get("package", {}).get("namespace") # Check for unmanaged flag on a namespaced package unmanaged = namespace and dependency.get("unmanaged") is True # Look for subfolders under unpackaged/pre unpackaged_pre = [] try: contents = repo.directory_contents("unpackaged/pre", return_as=dict, **kwargs) except NotFoundError: contents = None if contents: for dirname in list(contents.keys()): subfolder = "unpackaged/pre/{}".format(dirname) if subfolder in skip: continue unpackaged_pre.append({ "repo_owner": repo_owner, "repo_name": repo_name, "ref": tag, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), }) # Look for metadata under src (deployed if no namespace) unmanaged_src = None if unmanaged or not namespace: contents = repo.directory_contents("src", **kwargs) if contents: subfolder = "src" unmanaged_src = { "repo_owner": repo_owner, "repo_name": repo_name, "ref": tag, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), } # Look for subfolders under unpackaged/post unpackaged_post = [] try: contents = repo.directory_contents("unpackaged/post", return_as=dict, **kwargs) except NotFoundError: contents = None if contents: for dirname in list(contents.keys()): subfolder = "unpackaged/post/{}".format(dirname) if subfolder in skip: continue dependency = { "repo_owner": repo_owner, "repo_name": repo_name, "ref": tag, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), } # By default, we always inject the project's namespace into # unpackaged/post metadata if namespace and not dependency.get("namespace_inject"): dependency["namespace_inject"] = namespace dependency["unmanaged"] = unmanaged unpackaged_post.append(dependency) # Parse values from the repo's cumulusci.yml project = cumulusci_yml.get("project", {}) prefix_beta = project.get("git", {}).get("prefix_beta", "beta/") prefix_release = project.get("git", {}).get("prefix_release", "release/") dependencies = project.get("dependencies") if dependencies: dependencies = self.get_static_dependencies( dependencies, include_beta=include_beta) # Create the final ordered list of all parsed dependencies repo_dependencies = [] # unpackaged/pre/* if unpackaged_pre: repo_dependencies.extend(unpackaged_pre) if namespace and not unmanaged: version = None if tag: version = self.get_version_for_tag(tag, prefix_beta, prefix_release) else: version = self._find_release_version(repo, indent, include_beta) if not version: raise DependencyResolutionError( "{}Could not find latest release for {}".format( indent, namespace)) # If a latest prod version was found, make the dependencies a # child of that install dependency = {"namespace": namespace, "version": version} if dependencies: dependency["dependencies"] = dependencies repo_dependencies.append(dependency) # Unmanaged metadata from src (if referenced repo doesn't have a # namespace) else: if dependencies: repo_dependencies.extend(dependencies) if unmanaged_src: repo_dependencies.append(unmanaged_src) # unpackaged/post/* if unpackaged_post: repo_dependencies.extend(unpackaged_post) return repo_dependencies
def process_github_dependency( # noqa: C901 self, dependency, indent=None, include_beta=None ): if not indent: indent = "" self.logger.info( "{}Processing dependencies from Github repo {}".format( indent, dependency["github"] ) ) skip = dependency.get("skip") if not isinstance(skip, list): skip = [skip] # Initialize github3.py API against repo gh = self.get_github_api() repo_owner, repo_name = dependency["github"].split("/")[3:5] if repo_name.endswith(".git"): repo_name = repo_name[:-4] repo = gh.repository(repo_owner, repo_name) # Determine the commit release = None if "ref" in dependency: ref = dependency["ref"] else: if "tag" in dependency: try: # Find the github release corresponding to this tag. release = repo.release_from_tag(dependency["tag"]) except NotFoundError: raise DependencyResolutionError( "{}No release found for tag {}".format( indent, dependency["tag"] ) ) else: release = self._find_latest_release(repo, include_beta) if release: ref = repo.tag( repo.ref("tags/" + release.tag_name).object.sha ).object.sha else: self.logger.info( "{}No release found; using the latest commit from the {} branch.".format( indent, repo.default_branch ) ) ref = repo.branch(repo.default_branch).commit.sha # Get the cumulusci.yml file contents = repo.file_contents("cumulusci.yml", ref=ref) cumulusci_yml = ordered_yaml_load(contents.decoded) # Get the namespace from the cumulusci.yml if set package_config = cumulusci_yml.get("project", {}).get("package", {}) namespace = package_config.get("namespace") package_name = ( package_config.get("name_managed") or package_config.get("name") or namespace ) # Check for unmanaged flag on a namespaced package unmanaged = namespace and dependency.get("unmanaged") is True # Look for subfolders under unpackaged/pre unpackaged_pre = [] try: contents = repo.directory_contents( "unpackaged/pre", return_as=dict, ref=ref ) except NotFoundError: contents = None if contents: for dirname in list(contents.keys()): subfolder = "unpackaged/pre/{}".format(dirname) if subfolder in skip: continue name = "Deploy {}".format(subfolder) unpackaged_pre.append( { "name": name, "repo_owner": repo_owner, "repo_name": repo_name, "ref": ref, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), } ) # Look for metadata under src (deployed if no namespace) unmanaged_src = None if unmanaged or not namespace: contents = repo.directory_contents("src", ref=ref) if contents: subfolder = "src" unmanaged_src = { "name": "Deploy {}".format(package_name or repo_name), "repo_owner": repo_owner, "repo_name": repo_name, "ref": ref, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), } # Look for subfolders under unpackaged/post unpackaged_post = [] try: contents = repo.directory_contents( "unpackaged/post", return_as=dict, ref=ref ) except NotFoundError: contents = None if contents: for dirname in list(contents.keys()): subfolder = "unpackaged/post/{}".format(dirname) if subfolder in skip: continue name = "Deploy {}".format(subfolder) dependency = { "name": name, "repo_owner": repo_owner, "repo_name": repo_name, "ref": ref, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), } # By default, we always inject the project's namespace into # unpackaged/post metadata if namespace and not dependency.get("namespace_inject"): dependency["namespace_inject"] = namespace dependency["unmanaged"] = unmanaged unpackaged_post.append(dependency) # Parse values from the repo's cumulusci.yml project = cumulusci_yml.get("project", {}) dependencies = project.get("dependencies") if dependencies: dependencies = self.get_static_dependencies( dependencies, include_beta=include_beta ) # Create the final ordered list of all parsed dependencies repo_dependencies = [] # unpackaged/pre/* if unpackaged_pre: repo_dependencies.extend(unpackaged_pre) if namespace and not unmanaged: if release is None: raise DependencyResolutionError( "{}Could not find latest release for {}".format(indent, namespace) ) version = release.name # If a latest prod version was found, make the dependencies a # child of that install dependency = { "name": "Install {} {}".format(package_name or namespace, version), "namespace": namespace, "version": version, } if dependencies: dependency["dependencies"] = dependencies repo_dependencies.append(dependency) # Unmanaged metadata from src (if referenced repo doesn't have a # namespace) else: if dependencies: repo_dependencies.extend(dependencies) if unmanaged_src: repo_dependencies.append(unmanaged_src) # unpackaged/post/* if unpackaged_post: repo_dependencies.extend(unpackaged_post) return repo_dependencies
def process_github_dependency( # noqa: C901 self, dependency, indent=None, include_beta=None): if not indent: indent = "" self.logger.info( "{}Processing dependencies from Github repo {}".format( indent, dependency["github"])) skip = dependency.get("skip") if not isinstance(skip, list): skip = [skip] # Initialize github3.py API against repo repo_owner, repo_name = dependency["github"].split("/")[3:5] if repo_name.endswith(".git"): repo_name = repo_name[:-4] gh = self.get_github_api(repo_owner, repo_name) repo = gh.repository(repo_owner, repo_name) # Determine the commit release = None if "ref" in dependency: ref = dependency["ref"] else: if "tag" in dependency: try: # Find the github release corresponding to this tag. release = repo.release_from_tag(dependency["tag"]) except NotFoundError: raise DependencyResolutionError( "{}No release found for tag {}".format( indent, dependency["tag"])) else: release = self._find_latest_release(repo, include_beta) if release: ref = repo.tag( repo.ref("tags/" + release.tag_name).object.sha).object.sha else: self.logger.info( "{}No release found; using the latest commit from the {} branch." .format(indent, repo.default_branch)) ref = repo.branch(repo.default_branch).commit.sha # Get the cumulusci.yml file contents = repo.file_contents("cumulusci.yml", ref=ref) cumulusci_yml = ordered_yaml_load(contents.decoded) # Get the namespace from the cumulusci.yml if set package_config = cumulusci_yml.get("project", {}).get("package", {}) namespace = package_config.get("namespace") package_name = (package_config.get("name_managed") or package_config.get("name") or namespace) # Check for unmanaged flag on a namespaced package unmanaged = namespace and dependency.get("unmanaged") is True # Look for subfolders under unpackaged/pre unpackaged_pre = [] try: contents = repo.directory_contents("unpackaged/pre", return_as=dict, ref=ref) except NotFoundError: contents = None if contents: for dirname in list(contents.keys()): subfolder = "unpackaged/pre/{}".format(dirname) if subfolder in skip: continue name = "Deploy {}".format(subfolder) unpackaged_pre.append({ "name": name, "repo_owner": repo_owner, "repo_name": repo_name, "ref": ref, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), }) # Look for metadata under src (deployed if no namespace) unmanaged_src = None if unmanaged or not namespace: contents = repo.directory_contents("src", ref=ref) if contents: subfolder = "src" unmanaged_src = { "name": "Deploy {}".format(package_name or repo_name), "repo_owner": repo_owner, "repo_name": repo_name, "ref": ref, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), } # Look for subfolders under unpackaged/post unpackaged_post = [] try: contents = repo.directory_contents("unpackaged/post", return_as=dict, ref=ref) except NotFoundError: contents = None if contents: for dirname in list(contents.keys()): subfolder = "unpackaged/post/{}".format(dirname) if subfolder in skip: continue name = "Deploy {}".format(subfolder) dependency = { "name": name, "repo_owner": repo_owner, "repo_name": repo_name, "ref": ref, "subfolder": subfolder, "unmanaged": dependency.get("unmanaged"), "namespace_tokenize": dependency.get("namespace_tokenize"), "namespace_inject": dependency.get("namespace_inject"), "namespace_strip": dependency.get("namespace_strip"), } # By default, we always inject the project's namespace into # unpackaged/post metadata if namespace and not dependency.get("namespace_inject"): dependency["namespace_inject"] = namespace dependency["unmanaged"] = unmanaged unpackaged_post.append(dependency) # Parse values from the repo's cumulusci.yml project = cumulusci_yml.get("project", {}) dependencies = project.get("dependencies") if dependencies: dependencies = self.get_static_dependencies( dependencies, include_beta=include_beta) # Create the final ordered list of all parsed dependencies repo_dependencies = [] # unpackaged/pre/* if unpackaged_pre: repo_dependencies.extend(unpackaged_pre) if namespace and not unmanaged: if release is None: raise DependencyResolutionError( "{}Could not find latest release for {}".format( indent, namespace)) version = release.name # If a latest prod version was found, make the dependencies a # child of that install dependency = { "name": "Install {} {}".format(package_name or namespace, version), "namespace": namespace, "version": version, } if dependencies: dependency["dependencies"] = dependencies repo_dependencies.append(dependency) # Unmanaged metadata from src (if referenced repo doesn't have a # namespace) else: if dependencies: repo_dependencies.extend(dependencies) if unmanaged_src: repo_dependencies.append(unmanaged_src) # unpackaged/post/* if unpackaged_post: repo_dependencies.extend(unpackaged_post) return repo_dependencies