def test_simple_load(self, caplog):
yaml = """xyz:
y: abc"""
cciyml = cci_safe_load(StringIO(yaml))
assert not caplog.text
assert isinstance(cciyml, dict) # should parse despite funny character
assert cciyml["xyz"]["y"] == "abc", cciyml
def test_convert_nbsp(self, caplog):
yaml = """xyz:
\u00A0 y: abc"""
cciyml = cci_safe_load(StringIO(yaml))
assert "space character" in caplog.text
assert isinstance(cciyml, dict) # should parse despite funny character
assert cciyml["xyz"]["y"] == "abc", cciyml
def _get_repo_dependencies(self,
dependencies=None,
include_beta=None,
visited_repos=None):
"""Return a list of Package objects representing all of the GitHub repositories
in this project's dependency tree. Ignore all non-GitHub dependencies."""
deps = []
visited = visited_repos or set()
for dependency in dependencies:
if "github" in dependency:
repo = self.project_config.get_repo_from_url(
dependency["github"])
if repo is None:
raise DependencyResolutionError(
f"Github repository {dependency['github']} not found or not authorized."
)
_, ref = self.project_config.get_ref_for_dependency(
repo, dependency, include_beta)
contents = repo.file_contents("cumulusci.yml", ref=ref)
cumulusci_yml = cci_safe_load(
io.StringIO(contents.decoded.decode("utf-8")))
project = cumulusci_yml.get("project", {})
namespace = project.get("package", {}).get("namespace", "")
if namespace:
namespace = f"{namespace}__"
else:
namespace = ""
if f"{repo.owner}/{repo.name}" in visited:
continue
deps.append(
Package(
repo,
project.get("package",
{}).get("name",
f"{repo.owner}/{repo.name}"),
namespace,
project.get("git", {}).get("prefix_release",
"release/"),
))
visited.add(f"{repo.owner}/{repo.name}")
deps.extend(
self._get_repo_dependencies(
cumulusci_yml.get("project",
{}).get("dependencies", []),
visited_repos=visited,
))
return deps
def process_github_dependency( # noqa: C901
self, dependency, indent=None, include_beta=None):
if not indent:
indent = ""
self.logger.info(
f"{indent}Processing dependencies from Github repo {dependency['github']}"
)
skip = dependency.get("skip")
if not isinstance(skip, list):
skip = [skip]
# Initialize github3.py API against repo
repo_owner, repo_name = dependency["github"].split("/")[3:5]
if repo_name.endswith(".git"):
repo_name = repo_name[:-4]
gh = self.get_github_api(repo_owner, repo_name)
repo = gh.repository(repo_owner, repo_name)
if repo is None:
raise DependencyResolutionError(
f"{indent}Github repository {dependency['github']} not found or not authorized."
)
# Determine the commit
release = None
if "ref" in dependency:
ref = dependency["ref"]
else:
if "tag" in dependency:
try:
# Find the github release corresponding to this tag.
release = repo.release_from_tag(dependency["tag"])
except NotFoundError:
raise DependencyResolutionError(
f"{indent}No release found for tag {dependency['tag']}"
)
else:
release = find_latest_release(repo, include_beta)
if release:
ref = repo.tag(
repo.ref("tags/" + release.tag_name).object.sha).object.sha
else:
self.logger.info(
f"{indent}No release found; using the latest commit from the {repo.default_branch} branch."
)
ref = repo.branch(repo.default_branch).commit.sha
# Get the cumulusci.yml file
contents = repo.file_contents("cumulusci.yml", ref=ref)
cumulusci_yml = cci_safe_load(
io.StringIO(contents.decoded.decode("utf-8")))
# Get the namespace from the cumulusci.yml if set
package_config = cumulusci_yml.get("project", {}).get("package", {})
namespace = package_config.get("namespace")
package_name = (package_config.get("name_managed")
or package_config.get("name") or namespace)
# Check for unmanaged flag on a namespaced package
unmanaged = namespace and dependency.get("unmanaged") is True
# Look for subfolders under unpackaged/pre
unpackaged_pre = []
try:
contents = repo.directory_contents("unpackaged/pre",
return_as=dict,
ref=ref)
except NotFoundError:
contents = None
if contents:
for dirname in list(contents.keys()):
subfolder = f"unpackaged/pre/{dirname}"
if subfolder in skip:
continue
name = f"Deploy {subfolder}"
unpackaged_pre.append({
"name":
name,
"repo_owner":
repo_owner,
"repo_name":
repo_name,
"ref":
ref,
"subfolder":
subfolder,
"unmanaged":
dependency.get("unmanaged"),
"namespace_tokenize":
dependency.get("namespace_tokenize"),
"namespace_inject":
dependency.get("namespace_inject"),
"namespace_strip":
dependency.get("namespace_strip"),
})
# Look for metadata under src (deployed if no namespace)
unmanaged_src = None
if unmanaged or not namespace:
contents = repo.directory_contents("src", ref=ref)
if contents:
subfolder = "src"
unmanaged_src = {
"name": f"Deploy {package_name or repo_name}",
"repo_owner": repo_owner,
"repo_name": repo_name,
"ref": ref,
"subfolder": subfolder,
"unmanaged": dependency.get("unmanaged"),
"namespace_tokenize": dependency.get("namespace_tokenize"),
"namespace_inject": dependency.get("namespace_inject"),
"namespace_strip": dependency.get("namespace_strip"),
}
# Look for subfolders under unpackaged/post
unpackaged_post = []
try:
contents = repo.directory_contents("unpackaged/post",
return_as=dict,
ref=ref)
except NotFoundError:
contents = None
if contents:
for dirname in list(contents.keys()):
subfolder = f"unpackaged/post/{dirname}"
if subfolder in skip:
continue
name = f"Deploy {subfolder}"
dependency = {
"name": name,
"repo_owner": repo_owner,
"repo_name": repo_name,
"ref": ref,
"subfolder": subfolder,
"unmanaged": dependency.get("unmanaged"),
"namespace_tokenize": dependency.get("namespace_tokenize"),
"namespace_inject": dependency.get("namespace_inject"),
"namespace_strip": dependency.get("namespace_strip"),
}
# By default, we always inject the project's namespace into
# unpackaged/post metadata
if namespace and not dependency.get("namespace_inject"):
dependency["namespace_inject"] = namespace
dependency["unmanaged"] = unmanaged
unpackaged_post.append(dependency)
# Parse values from the repo's cumulusci.yml
project = cumulusci_yml.get("project", {})
dependencies = project.get("dependencies")
if dependencies:
dependencies = self.get_static_dependencies(
dependencies, include_beta=include_beta)
# Create the final ordered list of all parsed dependencies
repo_dependencies = []
# unpackaged/pre/*
if unpackaged_pre:
repo_dependencies.extend(unpackaged_pre)
if namespace and not unmanaged:
if release is None:
raise DependencyResolutionError(
f"{indent}Could not find latest release for {namespace}")
version = release.name
# If a latest prod version was found, make the dependencies a
# child of that install
dependency = {
"name": f"Install {package_name or namespace} {version}",
"namespace": namespace,
"version": version,
}
if dependencies:
dependency["dependencies"] = dependencies
repo_dependencies.append(dependency)
# Unmanaged metadata from src (if referenced repo doesn't have a
# namespace)
else:
if dependencies:
repo_dependencies.extend(dependencies)
if unmanaged_src:
repo_dependencies.append(unmanaged_src)
# unpackaged/post/*
if unpackaged_post:
repo_dependencies.extend(unpackaged_post)
return repo_dependencies