def fetch(self, path=None): """Fetch the archive of the specified commit and construct its project config.""" # To do: copy this from a shared cache if path is None: path = os.path.join(".cci", "projects", self.repo_name, self.commit) if not os.path.exists(path): os.makedirs(path) zf = download_extract_github(self.gh, self.repo_owner, self.repo_name, ref=self.commit) try: zf.extractall(path) except Exception: # make sure we don't leave an incomplete cache shutil.rmtree(path) raise project_config = self.project_config.construct_subproject_config( repo_info={ "root": os.path.realpath(path), "owner": self.repo_owner, "name": self.repo_name, "url": self.url, "commit": self.commit, }) return project_config
def _run_task(self): # Find or create Version if not self.dry_run: product = self._find_product() version = self._find_or_create_version(product) # Check out the specified tag repo_owner = self.project_config.repo_owner repo_name = self.project_config.repo_name gh = self.project_config.get_github_api() repo = gh.repository(repo_owner, repo_name) if self.tag: tag = self.options["tag"] self.commit = repo.tag(repo.ref("tags/" + tag).object.sha).object.sha self.logger.info("Downloading commit {} of {} from GitHub".format( self.commit, repo.full_name)) zf = download_extract_github(gh, repo_owner, repo_name, ref=self.commit) with temporary_dir() as project_dir: zf.extractall(project_dir) project_config = BaseProjectConfig( self.project_config.global_config_obj, repo_info={ "root": project_dir, "owner": repo_owner, "name": repo_name, "url": self.project_config.repo_url, "branch": self.tag or self.commit, "commit": self.commit, }, ) project_config.set_keychain(self.project_config.keychain) # Create each plan for plan_name, plan_config in self.plan_configs.items(): steps = self._freeze_steps(project_config, plan_config) self.logger.debug("Prepared steps:\n" + json.dumps(steps, indent=4)) if not self.dry_run: self._publish_plan(product, version, plan_name, plan_config, steps) # Update version to set is_listed=True if self.publish: self._call_api( "PATCH", "/versions/{}".format(version["id"]), json={"is_listed": True}, ) self.logger.info("Published Version {}".format(version["url"]))
def local_github_checkout(repo_owner, repo_name, commit_ish=None): with temporary_dir() as repo_root: # pretend it's a git clone to satisfy cci os.mkdir(".git") repo = get_github_api_for_repo(None, repo_owner, repo_name) if commit_ish is None: commit_ish = repo.repository(repo_owner, repo_name).default_branch zip_file = download_extract_github(repo, repo_owner, repo_name, ref=commit_ish) zip_file.extractall(repo_root) yield repo_root
def _run_task(self): # Find or create Version if not self.dry_run: product = self._find_product() version = self._find_or_create_version(product) # Check out the specified tag repo_owner = self.project_config.repo_owner repo_name = self.project_config.repo_name gh = self.project_config.get_github_api() repo = gh.repository(repo_owner, repo_name) tag = self.options["tag"] commit_sha = repo.tag(repo.ref("tags/" + tag).object.sha).object.sha self.logger.info( "Downloading commit {} of {} from GitHub".format(commit_sha, repo.full_name) ) zf = download_extract_github(gh, repo_owner, repo_name, ref=commit_sha) with temporary_dir() as project_dir: zf.extractall(project_dir) project_config = BaseProjectConfig( self.project_config.global_config_obj, repo_info={ "root": project_dir, "owner": repo_owner, "name": repo_name, "url": self.project_config.repo_url, "branch": tag, "commit": commit_sha, }, ) project_config.set_keychain(self.project_config.keychain) # Create each plan for plan_name, plan_config in self.plan_configs.items(): steps = self._freeze_steps(project_config, plan_config) self.logger.debug("Prepared steps:\n" + json.dumps(steps, indent=4)) if not self.dry_run: self._publish_plan(product, version, plan_name, plan_config, steps) # Update version to set is_listed=True if not self.dry_run: self._call_api( "PATCH", "/versions/{}".format(version["id"]), json={"is_listed": True}, ) self.logger.info("Published Version {}".format(version["url"]))
def test_download_extract_github(self): f = io.BytesIO() with zipfile.ZipFile(f, "w") as zf: zf.writestr("top/", "top") zf.writestr("top/src/", "top_src") zf.writestr("top/src/test", "test") f.seek(0) zipbytes = f.read() mock_repo = mock.Mock(default_branch="master") def assign_bytes(archive_type, zip_content, ref=None): zip_content.write(zipbytes) mock_archive = mock.Mock(return_value=True, side_effect=assign_bytes) mock_repo.archive = mock_archive zf = utils.download_extract_github(mock_repo, "src") result = zf.read("test") self.assertEqual(b"test", result)
def test_download_extract_github(self): f = io.BytesIO() with zipfile.ZipFile(f, "w") as zf: zf.writestr("top/", "top") zf.writestr("top/src/", "top_src") zf.writestr("top/src/test", "test") f.seek(0) zipbytes = f.read() mock_repo = mock.Mock(default_branch="master") mock_github = mock.Mock() mock_github.repository.return_value = mock_repo def assign_bytes(archive_type, zip_content, ref=None): zip_content.write(zipbytes) mock_archive = mock.Mock(return_value=True, side_effect=assign_bytes) mock_repo.archive = mock_archive zf = utils.download_extract_github(mock_github, "TestOwner", "TestRepo", "src") result = zf.read("test") self.assertEqual(b"test", result)
def _create_unlocked_package_from_github(self, dependency, dependencies): gh_for_repo = self.project_config.get_github_api( dependency["repo_owner"], dependency["repo_name"]) zip_src = download_extract_github( gh_for_repo, dependency["repo_owner"], dependency["repo_name"], dependency["subfolder"], ref=dependency.get("ref"), ) package_zip_builder = MetadataPackageZipBuilder.from_zipfile( zip_src, options=dependency, logger=self.logger) package_config = PackageConfig( package_name="{repo_owner}/{repo_name} {subfolder}".format( **dependency), version_name="Auto", package_type="Unlocked", # Ideally we'd do this without a namespace, # but it needs to match the dependent package namespace=self.package_config.namespace, ) package_id = self._get_or_create_package(package_config) self.request_id = self._create_version_request( package_id, package_config, package_zip_builder, dependencies=dependencies, ) self._poll() self._reset_poll() res = self.tooling.query( "SELECT SubscriberPackageVersionId FROM Package2Version " f"WHERE Id='{self.package_version_id}'") package2_version = res["records"][0] return package2_version["SubscriberPackageVersionId"]
def fetch(self, path=None): """Fetch the archive of the specified commit and construct its project config.""" # To do: copy this from a shared cache if path is None: path = os.path.join(".cci", "projects", self.repo_name, self.commit) if not os.path.exists(path): os.makedirs(path) zf = download_extract_github( self.gh, self.repo_owner, self.repo_name, ref=self.commit ) zf.extractall(path) project_config = self.project_config.__class__( self.project_config.global_config_obj, repo_info={ "root": os.path.realpath(path), "owner": self.repo_owner, "name": self.repo_name, "url": self.url, "commit": self.commit, }, included_sources=self.project_config.included_sources, ) return project_config
def _install_dependency(self, dependency): if "zip_url" or "repo" in dependency: package_zip = None if "zip_url" in dependency: self.logger.info( "Deploying unmanaged metadata from /{} of {}".format( dependency["subfolder"], dependency["zip_url"])) package_zip = download_extract_zip( dependency["zip_url"], subfolder=dependency.get("subfolder")) elif "repo" in dependency: self.logger.info( "Deploying unmanaged metadata from /{} of {}".format( dependency["subfolder"], dependency["repo"].full_name)) package_zip = download_extract_github( dependency["repo"], dependency["subfolder"], ref=dependency.get("ref"), ) if package_zip: if dependency.get("namespace_tokenize"): self.logger.info( "Replacing namespace prefix {}__ in files and filenames with namespace token strings" .format("{}__".format( dependency["namespace_tokenize"]))) package_zip = zip_tokenize_namespace( package_zip, namespace=dependency["namespace_tokenize"], logger=self.logger, ) if dependency.get("namespace_inject"): self.logger.info( "Replacing namespace tokens with {}".format( "{}__".format(dependency["namespace_inject"]))) package_zip = zip_inject_namespace( package_zip, namespace=dependency["namespace_inject"], managed=not dependency.get("unmanaged"), namespaced_org=self.options["namespaced_org"], logger=self.logger, ) if dependency.get("namespace_strip"): self.logger.info( "Removing namespace prefix {}__ from all files and filenames" .format("{}__".format(dependency["namespace_strip"]))) package_zip = zip_strip_namespace( package_zip, namespace=dependency["namespace_strip"], logger=self.logger, ) package_zip = ZipfilePackageZipBuilder(package_zip)() elif "namespace" in dependency: self.logger.info("Installing {} version {}".format( dependency["namespace"], dependency["version"])) package_zip = InstallPackageZipBuilder(dependency["namespace"], dependency["version"])() api = self.api_class(self, package_zip, purge_on_delete=self.options["purge_on_delete"]) return api()
def _run_task(self): tag = self.options["tag"] repo_owner = self.project_config.repo_owner repo_name = self.project_config.repo_name gh = self.project_config.get_github_api() repo = gh.repository(repo_owner, repo_name) commit_sha = repo.tag(repo.ref("tags/" + tag).object.sha).object.sha self.logger.info("Downloading commit {} of {}/{} from GitHub".format( commit_sha, repo_owner, repo_name)) zf = download_extract_github(gh, repo_owner, repo_name, ref=commit_sha) with temporary_dir() as project_dir: zf.extractall(project_dir) project_config = BaseProjectConfig( self.project_config.global_config_obj, repo_info={ "root": project_dir, "owner": repo_owner, "name": repo_name, "url": self.project_config.repo_url, "branch": tag, "commit": commit_sha, }, ) project_config.set_keychain(self.project_config.keychain) steps = self._freeze_steps(project_config) self.logger.debug("Publishing steps:\n" + json.dumps(steps, indent=4)) # create version (not listed yet) product_url = self.base_url + "/products/{}".format( self.options["product_id"]) label = self.project_config.get_version_for_tag(tag) version = self._call_api( "POST", "/versions", json={ "product": product_url, "label": label, "description": self.options.get("description", ""), "is_production": True, "commit_ish": self.project_config.repo_commit, "is_listed": False, }, ) self.logger.info("Created {}".format(version["url"])) # create plan plan_template_id = self.options.get("plan_template_id") plan_template_url = (self.base_url + "/plantemplates/{}".format(plan_template_id) if plan_template_id else None) plan = self._call_api( "POST", "/plans", json={ "plan_template": plan_template_url, "post_install_message_additional": self.options.get("post_install_message_additional", ""), "preflight_message_additional": self.options.get("preflight_message_additional", ""), "steps": steps, "tier": self.options["tier"], "title": self.options["title"], "version": version["url"], }, ) self.logger.info("Created Plan {}".format(plan["url"])) # create plan slug planslug = self._call_api( "POST", "/planslug", json={ "parent": plan["url"], "slug": self.options["slug"] }, ) self.logger.info("Created PlanSlug {}".format(planslug["url"])) # update version to set is_listed=True self._call_api("PATCH", "/versions/{}".format(version["id"]), json={"is_listed": True}) self.logger.info("Published Version {}".format(version["url"]))
def _run_task(self): # Find or create Version product = self._find_product() if not self.dry_run: version = self._find_or_create_version(product) if self.labels_path and "slug" in product: self._publish_labels(product["slug"]) # Check out the specified tag repo_owner = self.project_config.repo_owner repo_name = self.project_config.repo_name gh = self.project_config.get_github_api() repo = gh.repository(repo_owner, repo_name) if self.tag: tag = self.options["tag"] self.commit = repo.tag(repo.ref("tags/" + tag).object.sha).object.sha self.logger.info( "Downloading commit {} of {} from GitHub".format( self.commit, repo.full_name ) ) zf = download_extract_github(gh, repo_owner, repo_name, ref=self.commit) with temporary_dir() as project_dir: zf.extractall(project_dir) project_config = BaseProjectConfig( self.project_config.universal_config_obj, repo_info={ "root": project_dir, "owner": repo_owner, "name": repo_name, "url": self.project_config.repo_url, "branch": self.tag or self.commit, "commit": self.commit, }, ) project_config.set_keychain(self.project_config.keychain) # Create each plan for plan_name, plan_config in self.plan_configs.items(): self._add_labels( plan_config, f"plan:{plan_name}", { "title": "title of installation plan", "preflight_message": "shown before user starts installation (markdown)", "preflight_message_additional": "shown before user starts installation (markdown)", "post_install_message": "shown after successful installation (markdown)", "post_install_message_additional": "shown after successful installation (markdown)", "error_message": "shown after failed installation (markdown)", }, ) checks = plan_config.get("checks") or [] for check in checks: self._add_label( "checks", check.get("message"), "shown if validation fails" ) steps = self._freeze_steps(project_config, plan_config) self.logger.debug("Prepared steps:\n" + json.dumps(steps, indent=4)) for step in steps: # avoid separate labels for installing each package if step["name"].startswith("Install "): self._add_label( "steps", "Install {product} {version}", "title of installation step", ) else: self._add_label( "steps", step["name"], "title of installation step" ) self._add_label( "steps", step.get("description"), "description of installation step", ) for check in step["task_config"].get("checks", []): self._add_label( "checks", check.get("message"), "shown if validation fails" ) if not self.dry_run: self._publish_plan(product, version, plan_name, plan_config, steps) # Update version to set is_listed=True if self.publish: self._call_api( "PATCH", "/versions/{}".format(version["id"]), json={"is_listed": True}, ) self.logger.info("Published Version {}".format(version["url"])) # Save labels self._save_labels()