def fetch(self, path=None):
"""Fetch the archive of the specified commit and construct its project config."""
# To do: copy this from a shared cache
if path is None:
path = os.path.join(".cci", "projects", self.repo_name,
self.commit)
if not os.path.exists(path):
os.makedirs(path)
zf = download_extract_github(self.gh,
self.repo_owner,
self.repo_name,
ref=self.commit)
try:
zf.extractall(path)
except Exception:
# make sure we don't leave an incomplete cache
shutil.rmtree(path)
raise
project_config = self.project_config.construct_subproject_config(
repo_info={
"root": os.path.realpath(path),
"owner": self.repo_owner,
"name": self.repo_name,
"url": self.url,
"commit": self.commit,
})
return project_config
def _run_task(self):
# Find or create Version
if not self.dry_run:
product = self._find_product()
version = self._find_or_create_version(product)
# Check out the specified tag
repo_owner = self.project_config.repo_owner
repo_name = self.project_config.repo_name
gh = self.project_config.get_github_api()
repo = gh.repository(repo_owner, repo_name)
if self.tag:
tag = self.options["tag"]
self.commit = repo.tag(repo.ref("tags/" +
tag).object.sha).object.sha
self.logger.info("Downloading commit {} of {} from GitHub".format(
self.commit, repo.full_name))
zf = download_extract_github(gh,
repo_owner,
repo_name,
ref=self.commit)
with temporary_dir() as project_dir:
zf.extractall(project_dir)
project_config = BaseProjectConfig(
self.project_config.global_config_obj,
repo_info={
"root": project_dir,
"owner": repo_owner,
"name": repo_name,
"url": self.project_config.repo_url,
"branch": self.tag or self.commit,
"commit": self.commit,
},
)
project_config.set_keychain(self.project_config.keychain)
# Create each plan
for plan_name, plan_config in self.plan_configs.items():
steps = self._freeze_steps(project_config, plan_config)
self.logger.debug("Prepared steps:\n" +
json.dumps(steps, indent=4))
if not self.dry_run:
self._publish_plan(product, version, plan_name,
plan_config, steps)
# Update version to set is_listed=True
if self.publish:
self._call_api(
"PATCH",
"/versions/{}".format(version["id"]),
json={"is_listed": True},
)
self.logger.info("Published Version {}".format(version["url"]))
def local_github_checkout(repo_owner, repo_name, commit_ish=None):
with temporary_dir() as repo_root:
# pretend it's a git clone to satisfy cci
os.mkdir(".git")
repo = get_github_api_for_repo(None, repo_owner, repo_name)
if commit_ish is None:
commit_ish = repo.repository(repo_owner, repo_name).default_branch
zip_file = download_extract_github(repo,
repo_owner,
repo_name,
ref=commit_ish)
zip_file.extractall(repo_root)
yield repo_root
def _run_task(self):
# Find or create Version
if not self.dry_run:
product = self._find_product()
version = self._find_or_create_version(product)
# Check out the specified tag
repo_owner = self.project_config.repo_owner
repo_name = self.project_config.repo_name
gh = self.project_config.get_github_api()
repo = gh.repository(repo_owner, repo_name)
tag = self.options["tag"]
commit_sha = repo.tag(repo.ref("tags/" + tag).object.sha).object.sha
self.logger.info(
"Downloading commit {} of {} from GitHub".format(commit_sha, repo.full_name)
)
zf = download_extract_github(gh, repo_owner, repo_name, ref=commit_sha)
with temporary_dir() as project_dir:
zf.extractall(project_dir)
project_config = BaseProjectConfig(
self.project_config.global_config_obj,
repo_info={
"root": project_dir,
"owner": repo_owner,
"name": repo_name,
"url": self.project_config.repo_url,
"branch": tag,
"commit": commit_sha,
},
)
project_config.set_keychain(self.project_config.keychain)
# Create each plan
for plan_name, plan_config in self.plan_configs.items():
steps = self._freeze_steps(project_config, plan_config)
self.logger.debug("Prepared steps:\n" + json.dumps(steps, indent=4))
if not self.dry_run:
self._publish_plan(product, version, plan_name, plan_config, steps)
# Update version to set is_listed=True
if not self.dry_run:
self._call_api(
"PATCH",
"/versions/{}".format(version["id"]),
json={"is_listed": True},
)
self.logger.info("Published Version {}".format(version["url"]))
def test_download_extract_github(self):
f = io.BytesIO()
with zipfile.ZipFile(f, "w") as zf:
zf.writestr("top/", "top")
zf.writestr("top/src/", "top_src")
zf.writestr("top/src/test", "test")
f.seek(0)
zipbytes = f.read()
mock_repo = mock.Mock(default_branch="master")
def assign_bytes(archive_type, zip_content, ref=None):
zip_content.write(zipbytes)
mock_archive = mock.Mock(return_value=True, side_effect=assign_bytes)
mock_repo.archive = mock_archive
zf = utils.download_extract_github(mock_repo, "src")
result = zf.read("test")
self.assertEqual(b"test", result)
def test_download_extract_github(self):
f = io.BytesIO()
with zipfile.ZipFile(f, "w") as zf:
zf.writestr("top/", "top")
zf.writestr("top/src/", "top_src")
zf.writestr("top/src/test", "test")
f.seek(0)
zipbytes = f.read()
mock_repo = mock.Mock(default_branch="master")
mock_github = mock.Mock()
mock_github.repository.return_value = mock_repo
def assign_bytes(archive_type, zip_content, ref=None):
zip_content.write(zipbytes)
mock_archive = mock.Mock(return_value=True, side_effect=assign_bytes)
mock_repo.archive = mock_archive
zf = utils.download_extract_github(mock_github, "TestOwner", "TestRepo", "src")
result = zf.read("test")
self.assertEqual(b"test", result)
def _create_unlocked_package_from_github(self, dependency, dependencies):
gh_for_repo = self.project_config.get_github_api(
dependency["repo_owner"], dependency["repo_name"])
zip_src = download_extract_github(
gh_for_repo,
dependency["repo_owner"],
dependency["repo_name"],
dependency["subfolder"],
ref=dependency.get("ref"),
)
package_zip_builder = MetadataPackageZipBuilder.from_zipfile(
zip_src, options=dependency, logger=self.logger)
package_config = PackageConfig(
package_name="{repo_owner}/{repo_name} {subfolder}".format(
**dependency),
version_name="Auto",
package_type="Unlocked",
# Ideally we'd do this without a namespace,
# but it needs to match the dependent package
namespace=self.package_config.namespace,
)
package_id = self._get_or_create_package(package_config)
self.request_id = self._create_version_request(
package_id,
package_config,
package_zip_builder,
dependencies=dependencies,
)
self._poll()
self._reset_poll()
res = self.tooling.query(
"SELECT SubscriberPackageVersionId FROM Package2Version "
f"WHERE Id='{self.package_version_id}'")
package2_version = res["records"][0]
return package2_version["SubscriberPackageVersionId"]
def fetch(self, path=None):
"""Fetch the archive of the specified commit and construct its project config."""
# To do: copy this from a shared cache
if path is None:
path = os.path.join(".cci", "projects", self.repo_name, self.commit)
if not os.path.exists(path):
os.makedirs(path)
zf = download_extract_github(
self.gh, self.repo_owner, self.repo_name, ref=self.commit
)
zf.extractall(path)
project_config = self.project_config.__class__(
self.project_config.global_config_obj,
repo_info={
"root": os.path.realpath(path),
"owner": self.repo_owner,
"name": self.repo_name,
"url": self.url,
"commit": self.commit,
},
included_sources=self.project_config.included_sources,
)
return project_config
def _install_dependency(self, dependency):
if "zip_url" or "repo" in dependency:
package_zip = None
if "zip_url" in dependency:
self.logger.info(
"Deploying unmanaged metadata from /{} of {}".format(
dependency["subfolder"], dependency["zip_url"]))
package_zip = download_extract_zip(
dependency["zip_url"],
subfolder=dependency.get("subfolder"))
elif "repo" in dependency:
self.logger.info(
"Deploying unmanaged metadata from /{} of {}".format(
dependency["subfolder"], dependency["repo"].full_name))
package_zip = download_extract_github(
dependency["repo"],
dependency["subfolder"],
ref=dependency.get("ref"),
)
if package_zip:
if dependency.get("namespace_tokenize"):
self.logger.info(
"Replacing namespace prefix {}__ in files and filenames with namespace token strings"
.format("{}__".format(
dependency["namespace_tokenize"])))
package_zip = zip_tokenize_namespace(
package_zip,
namespace=dependency["namespace_tokenize"],
logger=self.logger,
)
if dependency.get("namespace_inject"):
self.logger.info(
"Replacing namespace tokens with {}".format(
"{}__".format(dependency["namespace_inject"])))
package_zip = zip_inject_namespace(
package_zip,
namespace=dependency["namespace_inject"],
managed=not dependency.get("unmanaged"),
namespaced_org=self.options["namespaced_org"],
logger=self.logger,
)
if dependency.get("namespace_strip"):
self.logger.info(
"Removing namespace prefix {}__ from all files and filenames"
.format("{}__".format(dependency["namespace_strip"])))
package_zip = zip_strip_namespace(
package_zip,
namespace=dependency["namespace_strip"],
logger=self.logger,
)
package_zip = ZipfilePackageZipBuilder(package_zip)()
elif "namespace" in dependency:
self.logger.info("Installing {} version {}".format(
dependency["namespace"], dependency["version"]))
package_zip = InstallPackageZipBuilder(dependency["namespace"],
dependency["version"])()
api = self.api_class(self,
package_zip,
purge_on_delete=self.options["purge_on_delete"])
return api()
def _run_task(self):
tag = self.options["tag"]
repo_owner = self.project_config.repo_owner
repo_name = self.project_config.repo_name
gh = self.project_config.get_github_api()
repo = gh.repository(repo_owner, repo_name)
commit_sha = repo.tag(repo.ref("tags/" + tag).object.sha).object.sha
self.logger.info("Downloading commit {} of {}/{} from GitHub".format(
commit_sha, repo_owner, repo_name))
zf = download_extract_github(gh, repo_owner, repo_name, ref=commit_sha)
with temporary_dir() as project_dir:
zf.extractall(project_dir)
project_config = BaseProjectConfig(
self.project_config.global_config_obj,
repo_info={
"root": project_dir,
"owner": repo_owner,
"name": repo_name,
"url": self.project_config.repo_url,
"branch": tag,
"commit": commit_sha,
},
)
project_config.set_keychain(self.project_config.keychain)
steps = self._freeze_steps(project_config)
self.logger.debug("Publishing steps:\n" + json.dumps(steps, indent=4))
# create version (not listed yet)
product_url = self.base_url + "/products/{}".format(
self.options["product_id"])
label = self.project_config.get_version_for_tag(tag)
version = self._call_api(
"POST",
"/versions",
json={
"product": product_url,
"label": label,
"description": self.options.get("description", ""),
"is_production": True,
"commit_ish": self.project_config.repo_commit,
"is_listed": False,
},
)
self.logger.info("Created {}".format(version["url"]))
# create plan
plan_template_id = self.options.get("plan_template_id")
plan_template_url = (self.base_url +
"/plantemplates/{}".format(plan_template_id)
if plan_template_id else None)
plan = self._call_api(
"POST",
"/plans",
json={
"plan_template":
plan_template_url,
"post_install_message_additional":
self.options.get("post_install_message_additional", ""),
"preflight_message_additional":
self.options.get("preflight_message_additional", ""),
"steps":
steps,
"tier":
self.options["tier"],
"title":
self.options["title"],
"version":
version["url"],
},
)
self.logger.info("Created Plan {}".format(plan["url"]))
# create plan slug
planslug = self._call_api(
"POST",
"/planslug",
json={
"parent": plan["url"],
"slug": self.options["slug"]
},
)
self.logger.info("Created PlanSlug {}".format(planslug["url"]))
# update version to set is_listed=True
self._call_api("PATCH",
"/versions/{}".format(version["id"]),
json={"is_listed": True})
self.logger.info("Published Version {}".format(version["url"]))
def _run_task(self):
# Find or create Version
product = self._find_product()
if not self.dry_run:
version = self._find_or_create_version(product)
if self.labels_path and "slug" in product:
self._publish_labels(product["slug"])
# Check out the specified tag
repo_owner = self.project_config.repo_owner
repo_name = self.project_config.repo_name
gh = self.project_config.get_github_api()
repo = gh.repository(repo_owner, repo_name)
if self.tag:
tag = self.options["tag"]
self.commit = repo.tag(repo.ref("tags/" + tag).object.sha).object.sha
self.logger.info(
"Downloading commit {} of {} from GitHub".format(
self.commit, repo.full_name
)
)
zf = download_extract_github(gh, repo_owner, repo_name, ref=self.commit)
with temporary_dir() as project_dir:
zf.extractall(project_dir)
project_config = BaseProjectConfig(
self.project_config.universal_config_obj,
repo_info={
"root": project_dir,
"owner": repo_owner,
"name": repo_name,
"url": self.project_config.repo_url,
"branch": self.tag or self.commit,
"commit": self.commit,
},
)
project_config.set_keychain(self.project_config.keychain)
# Create each plan
for plan_name, plan_config in self.plan_configs.items():
self._add_labels(
plan_config,
f"plan:{plan_name}",
{
"title": "title of installation plan",
"preflight_message": "shown before user starts installation (markdown)",
"preflight_message_additional": "shown before user starts installation (markdown)",
"post_install_message": "shown after successful installation (markdown)",
"post_install_message_additional": "shown after successful installation (markdown)",
"error_message": "shown after failed installation (markdown)",
},
)
checks = plan_config.get("checks") or []
for check in checks:
self._add_label(
"checks", check.get("message"), "shown if validation fails"
)
steps = self._freeze_steps(project_config, plan_config)
self.logger.debug("Prepared steps:\n" + json.dumps(steps, indent=4))
for step in steps:
# avoid separate labels for installing each package
if step["name"].startswith("Install "):
self._add_label(
"steps",
"Install {product} {version}",
"title of installation step",
)
else:
self._add_label(
"steps", step["name"], "title of installation step"
)
self._add_label(
"steps",
step.get("description"),
"description of installation step",
)
for check in step["task_config"].get("checks", []):
self._add_label(
"checks", check.get("message"), "shown if validation fails"
)
if not self.dry_run:
self._publish_plan(product, version, plan_name, plan_config, steps)
# Update version to set is_listed=True
if self.publish:
self._call_api(
"PATCH",
"/versions/{}".format(version["id"]),
json={"is_listed": True},
)
self.logger.info("Published Version {}".format(version["url"]))
# Save labels
self._save_labels()
