def create_build_package(package_files, working_dir=None): """given a list of files, copy them to a temporary folder, compress into a .tar.gz, and rename based on the file hash. Return the full path to the .tar.gz in the temporary folder. Parameters ========== package_files: a list of files to include in the tar.gz working_dir: If set, the path derived for the recipe and files is relative to this. """ # Ensure package files all exist for package_file in package_files: if not os.path.exists(package_file): bot.exit("Cannot find %s." % package_file) bot.log("Generating build package for %s files..." % len(package_files)) build_dir = get_tmpdir(prefix="sregistry-build") build_tar = "%s/build.tar.gz" % build_dir tar = tarfile.open(build_tar, "w:gz") # Create the tar.gz, making sure relative to working_dir for package_file in package_files: # Get a relative path relative_path = get_relative_path(package_file, working_dir) tar.add(package_file, arcname=relative_path) tar.close() # Get hash (sha256), and rename file sha256 = get_file_hash(build_tar) hash_tar = "%s/%s.tar.gz" % (build_dir, sha256) shutil.move(build_tar, hash_tar) return hash_tar
def create_build_package(package_files): '''given a list of files, copy them to a temporary folder, compress into a .tar.gz, and rename based on the file hash. Return the full path to the .tar.gz in the temporary folder. Parameters ========== package_files: a list of files to include in the tar.gz ''' # Ensure package files all exist for package_file in package_files: if not os.path.exists(package_file): bot.exit('Cannot find %s.' % package_file) bot.log('Generating build package for %s files...' % len(package_files)) build_dir = get_tmpdir(prefix="sregistry-build") build_tar = '%s/build.tar.gz' % build_dir tar = tarfile.open(build_tar, "w:gz") # Create the tar.gz for package_file in package_files: tar.add(package_file) tar.close() # Get hash (sha256), and rename file sha256 = get_file_hash(build_tar) hash_tar = "%s/%s.tar.gz" % (build_dir, sha256) shutil.move(build_tar, hash_tar) return hash_tar
def build_status(self, build_id): """get a build status based on a build id. We return the entire response object for the client to parse. """ project = self._get_project() response = (self._build_service.projects().builds().get( id=build_id, projectId=project).execute()) build_id = response["id"] status = response["status"] bot.log("build %s: %s" % (build_id, status)) return response
def run_build(self, config): """run a build, meaning creating a build. Retry if there is failure """ response = self._submit_build(config) status = response["metadata"]["build"]["status"] build_id = response["metadata"]["build"]["id"] start = time.time() while status not in ["COMPLETE", "FAILURE", "SUCCESS", "TIMEOUT"]: time.sleep(15) response = self._build_status(build_id) status = response["status"] end = time.time() bot.log("Total build time: %s seconds" % (round(end - start, 2))) return self._finish_build(build_id, response=response, config=config)
def run_build(self, config): '''run a build, meaning creating a build. Retry if there is failure ''' response = self._submit_build(config) status = response['metadata']['build']['status'] build_id = response['metadata']['build']['id'] start = time.time() while status not in ['COMPLETE', 'FAILURE', 'SUCCESS', 'TIMEOUT']: time.sleep(15) response = self._build_status(build_id) status = response['status'] end = time.time() bot.log('Total build time: %s seconds' % (round(end - start, 2))) return self._finish_build(build_id, response=response, config=config)
def submit_build(self, config): """run a build, meaning creating a build. Retry if there is failure """ project = self._get_project() # prefix, message, color bot.custom("PROJECT", project, "CYAN") for i, step in enumerate(config["steps"]): bot.custom("BUILD %s" % i, step["name"], "CYAN") response = (self._build_service.projects().builds().create( body=config, projectId=project).execute()) build_id = response["metadata"]["build"]["id"] status = response["metadata"]["build"]["status"] bot.log("build %s: %s" % (build_id, status)) return response
def submit_build(self, config): '''run a build, meaning creating a build. Retry if there is failure ''' project = self._get_project() # prefix, message, color bot.custom('PROJECT', project, "CYAN") for i, step in enumerate(config['steps']): bot.custom('BUILD %s' % i, step['name'], "CYAN") response = self._build_service.projects().builds().create( body=config, projectId=project).execute() build_id = response['metadata']['build']['id'] status = response['metadata']['build']['status'] bot.log("build %s: %s" % (build_id, status)) return response
def run_build(self, config, bucket, names): '''run a build, meaning creating a build. Retry if there is failure ''' project = self._get_project() # prefix, message, color bot.custom('PROJECT', project, "CYAN") bot.custom('BUILD ', config['steps'][0]['name'], "CYAN") response = self._build_service.projects().builds().create( body=config, projectId=project).execute() build_id = response['metadata']['build']['id'] status = response['metadata']['build']['status'] bot.log("build %s: %s" % (build_id, status)) start = time.time() while status not in ['COMPLETE', 'FAILURE', 'SUCCESS']: time.sleep(15) response = self._build_service.projects().builds().get( id=build_id, projectId=project).execute() build_id = response['id'] status = response['status'] bot.log("build %s: %s" % (build_id, status)) end = time.time() bot.log('Total build time: %s seconds' % (round(end - start, 2))) # If successful, update blob metadata and visibility if status == 'SUCCESS': # Does the user want to keep the container private? env = 'SREGISTRY_GOOGLE_STORAGE_PRIVATE' blob = bucket.blob(response['artifacts']['objects']['paths'][0]) # Make Public, if desired if self._get_and_update_setting(env) == None: blob.make_public() response['public_url'] = blob.public_url # Add the metadata directly to the object update_blob_metadata(blob, response, config, bucket, names) response['media_link'] = blob.media_link response['size'] = blob.size response['file_hash'] = blob.md5_hash return response
def build( self, name, recipe="Singularity", context=None, preview=False, headless=False, working_dir=None, webhook=None, timeout=10800, extra_data=None, ): """trigger a build on Google Cloud (builder then storage) given a name recipe, and Github URI where the recipe can be found. This means creating and uploading a build package to use for the build. Parameters ========== recipe: the local recipe to build. name: should be the complete uri that the user has requested to push. context: the dependency files needed for the build. If not defined, only the recipe is uploaded. preview: if True, preview but don't run the build working_dir: The working directory for the build. Defaults to pwd. timeout: the number of seconds for the build to timeout. The default is 3 hours, and the maximum is 24 hours. If unset (None) it will be 10 minutes. webhook: if not None, add a curl POST to finish the build. headless: If true, don't track the build, but submit and provide an endpoint to send a response to. extra_data: a dictionary of extra_data to send back to the webhook (they are passed in the environment) Environment =========== SREGISTRY_GOOGLE_BUILD_SINGULARITY_VERSION: the version of Singularity to use, defaults to v3.2.1-slim SREGISTRY_GOOGLE_BUILD_CLEANUP: after build, delete intermediate dependencies in cloudbuild bucket. """ bot.debug("BUILD %s" % recipe) build_package = [recipe] if context: # If the user gives a ., include recursive $PWD if "." in context: context = glob(os.getcwd() + "/**/*", recursive=True) build_package = build_package + context # We need to get and save relative paths for the config. package = create_build_package(build_package, working_dir) # Does the package already exist? If the user cached, it might destination = "source/%s" % os.path.basename(package) blob = self._build_bucket.blob(destination) # if it doesn't exist, upload it if not blob.exists() and preview is False: bot.log("Uploading build package!") self._upload(source=package, bucket=self._build_bucket, destination=destination) else: bot.log("Build package found in %s." % self._build_bucket.name) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name)) prefix = "%s/" % names["collection"] # The name should include the complete uri so it's searchable name = os.path.basename(names["uri"]) # Update the recipe name to use a relative path recipe = get_relative_path(recipe, working_dir) # Load the build configuration (defaults to local) config = self._load_build_config(name=name, prefix=prefix, recipe=recipe) # Add a webhook, if defined if webhook and headless: config = add_webhook(config=config, webhook=webhook, extra_data=extra_data) # The source should point to the bucket with the .tar.gz, latest generation config["source"]["storageSource"]["bucket"] = self._build_bucket.name config["source"]["storageSource"]["object"] = destination # If the user wants a timeout if timeout is not None: config["timeout"] = "%ss" % timeout # If not a preview, run the build and return the response if not preview: if not headless: config = self._run_build(config) else: config = self._submit_build(config) # If the user wants to cache cloudbuild files, this will be set env = "SREGISTRY_GOOGLE_BUILD_CACHE" if not self._get_and_update_setting(env, self.envars.get(env)): if headless is False: blob.delete() # Clean up either way, return config or response shutil.rmtree(os.path.dirname(package)) return config
def build(self, name, recipe="Singularity", context=None, preview=False): '''trigger a build on Google Cloud (builder then storage) given a name recipe, and Github URI where the recipe can be found. Parameters ========== recipe: the local recipe to build. name: should be the complete uri that the user has requested to push. context: the dependency files needed for the build. If not defined, only the recipe is uploaded. preview: if True, preview but don't run the build Environment =========== SREGISTRY_GOOGLE_BUILD_SINGULARITY_VERSION: the version of Singularity to use, defaults to 3.0.2-slim SREGISTRY_GOOGLE_BUILD_CLEANUP: after build, delete intermediate dependencies in cloudbuild bucket. ''' bot.debug("BUILD %s" % recipe) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name)) # Load the build configuration config = self._load_build_config(name=names['uri'], recipe=recipe) build_package = [recipe] if context not in [None, '', []]: # If the user gives a ., include recursive $PWD if '.' in context: context = glob(os.getcwd() + '/**/*', recursive=True) build_package = build_package + context package = create_build_package(build_package) # Does the package already exist? If the user cached, it might destination = 'source/%s' % os.path.basename(package) blob = self._build_bucket.blob(destination) # if it doesn't exist, upload it if not blob.exists() and preview is False: bot.log('Uploading build package!') manifest = self._upload(source=package, bucket=self._build_bucket, destination=destination) else: bot.log('Build package found in %s.' % self._build_bucket.name) # The source should point to the bucket with the .tar.gz, latest generation config["source"]["storageSource"]['bucket'] = self._build_bucket.name config["source"]["storageSource"]['object'] = destination # If not a preview, run the build and return the response if preview is False: config = self._run_build(config, self._bucket, names) # If the user wants to cache cloudbuild files, this will be set if not self._get_and_update_setting('SREGISTRY_GOOGLE_BUILD_CACHE'): blob.delete() # Clean up either way, return config or response shutil.rmtree(os.path.dirname(package)) return config