def test_get_file_hash(): print("Testing utils.get_file_hash") from sregistry.utils import get_file_hash here = os.path.dirname(os.path.abspath(__file__)) testdata = os.path.join(here, "testdata", "hashtest.txt") assert (get_file_hash(testdata) == "6bb92117bded3da774363713657a629a9f38eac2e57cd47e1dcda21d3445c67d") assert get_file_hash(testdata, "md5") == "e5d376ca96081dd561ff303c3a631fd5"
def create_build_package(package_files, working_dir=None): """given a list of files, copy them to a temporary folder, compress into a .tar.gz, and rename based on the file hash. Return the full path to the .tar.gz in the temporary folder. Parameters ========== package_files: a list of files to include in the tar.gz working_dir: If set, the path derived for the recipe and files is relative to this. """ # Ensure package files all exist for package_file in package_files: if not os.path.exists(package_file): bot.exit("Cannot find %s." % package_file) bot.log("Generating build package for %s files..." % len(package_files)) build_dir = get_tmpdir(prefix="sregistry-build") build_tar = "%s/build.tar.gz" % build_dir tar = tarfile.open(build_tar, "w:gz") # Create the tar.gz, making sure relative to working_dir for package_file in package_files: # Get a relative path relative_path = get_relative_path(package_file, working_dir) tar.add(package_file, arcname=relative_path) tar.close() # Get hash (sha256), and rename file sha256 = get_file_hash(build_tar) hash_tar = "%s/%s.tar.gz" % (build_dir, sha256) shutil.move(build_tar, hash_tar) return hash_tar
def push(self, path, name, tag=None): """push an image to Google Cloud Storage, meaning uploading it Parameters ========== path: should correspond to an absolte image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker """ path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.exit("%s does not exist." % path) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name), tag=tag) if names["version"] is None: version = get_file_hash(path, "sha256") names = parse_image_name(remove_uri(name), tag=tag, version=version) # Update metadata with names metadata = self.get_metadata(path, names=names) manifest = self._upload(source=path, destination=names["storage"], metadata=metadata) print(manifest["mediaLink"])
def create_build_package(package_files): '''given a list of files, copy them to a temporary folder, compress into a .tar.gz, and rename based on the file hash. Return the full path to the .tar.gz in the temporary folder. Parameters ========== package_files: a list of files to include in the tar.gz ''' # Ensure package files all exist for package_file in package_files: if not os.path.exists(package_file): bot.exit('Cannot find %s.' % package_file) bot.log('Generating build package for %s files...' % len(package_files)) build_dir = get_tmpdir(prefix="sregistry-build") build_tar = '%s/build.tar.gz' % build_dir tar = tarfile.open(build_tar, "w:gz") # Create the tar.gz for package_file in package_files: tar.add(package_file) tar.close() # Get hash (sha256), and rename file sha256 = get_file_hash(build_tar) hash_tar = "%s/%s.tar.gz" % (build_dir, sha256) shutil.move(build_tar, hash_tar) return hash_tar
def calculate_version(cid): '''calculate version is run as a separate task after a container upload. Instead of using md5 provided by nginx we calculate sha256 sum and then include as the version variable. ''' from shub.apps.main.views import get_container from sregistry.utils import get_file_hash print("Calculating version for upload.") container = get_container(cid) version = "sha256.%s" % get_file_hash(container.image.datafile.path, "sha256") container.version = version container.save()
def push(self, path, name, tag=None): """push an image to Google Cloud Drive, meaning uploading it path: should correspond to an absolte image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker """ # The root of the drive for containers (the parent folder) parent = self._get_or_create_folder(self._base) image = None path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.exit("%s does not exist." % path) names = parse_image_name(remove_uri(name), tag=tag) if names["version"] is None: version = get_file_hash(path, "sha256") names = parse_image_name(remove_uri(name), tag=tag, version=version) # Update metadata with names, flatten to only include labels metadata = self.get_metadata(path, names=names) file_metadata = { "name": names["storage"], "mimeType": "application/octet-stream", "parents": [parent["id"]], "properties": metadata, } media = MediaFileUpload(path, resumable=True) try: bot.spinner.start() image = (self._service.files().create(body=file_metadata, media_body=media, fields="id").execute()) # Add a thumbnail! thumbnail = get_thumbnail() with open(thumbnail, "rb") as f: body = { "contentHints": { "thumbnail": { "image": base64.urlsafe_b64encode(f.read()).decode("utf8"), "mimeType": "image/png", } } } image = (self._service.files().update(fileId=image["id"], body=body).execute()) bot.spinner.stop() print(image["name"]) except HttpError: bot.error("Error uploading %s" % path) return image
def add(self, image_path=None, image_uri=None, image_name=None, url=None, metadata=None, save=True, copy=False): '''get or create a container, including the collection to add it to. This function can be used from a file on the local system, or via a URL that has been downloaded. Either way, if one of url, version, or image_file is not provided, the model is created without it. If a version is not provided but a file path is, then the file hash is used. Parameters ========== image_path: full path to image file image_name: if defined, the user wants a custom name (and not based on uri) metadata: any extra metadata to keep for the image (dict) save: if True, move the image to the cache if it's not there copy: If True, copy the image instead of moving it. image_name: a uri that gets parsed into a names object that looks like: {'collection': 'vsoch', 'image': 'hello-world', 'storage': 'vsoch/hello-world-latest.img', 'tag': 'latest', 'version': '12345' 'uri': 'vsoch/hello-world:latest@12345'} After running add, the user will take some image in a working directory, add it to the database, and have it available for search and use under SREGISTRY_STORAGE/<collection>/<container> If the container was retrieved from a webby place, it should have version If no version is found, the file hash is used. ''' from sregistry.database.models import Container # We can only save if the image is provided if image_path is not None: if not os.path.exists(image_path) and save is True: bot.exit('Cannot find %s' % image_path) # An image uri is required for version, tag, etc. if image_uri is None: bot.exit( 'You must provide an image uri --name <collection>/<namespace>') names = parse_image_name(remove_uri(image_uri)) bot.debug('Adding %s to registry' % names['uri']) # If Singularity is installed, inspect image for metadata metadata = self.get_metadata(image_path, names=names) collection = self.get_or_create_collection(names['collection']) # Get a hash of the file for the version, or use provided version = names.get('version') if not version: if image_path: version = get_file_hash(image_path, "sha256") else: version = '' # we can't determine a version, not in API/no file names = parse_image_name(remove_uri(image_uri), version=version) # If save, move to registry storage first if save and image_path: # If the user hasn't defined a custom name if image_name is None: image_name = self._get_storage_name(names) if copy: copyfile(image_path, image_name) else: shutil.move(image_path, image_name) image_path = image_name # Just in case the client didn't provide it, see if we have in metadata if url is None and "url" in metadata: url = metadata['url'] # First check that we don't have one already! container = self.get_container(name=names['image'], collection_id=collection.id, tag=names['tag'], version=version) # The container did not exist, create it if not container: action = "new" container = Container(metrics=json.dumps(metadata), name=names['image'], image=image_path, client=self.client_name, tag=names['tag'], version=version, url=url, uri=names['uri'], collection_id=collection.id) self.session.add(container) collection.containers.append(container) # The container existed, update it. else: action = "update" metrics = json.loads(container.metrics) metrics.update(metadata) container.url = url container.client = self.client_name if image_path is not None: container.image = image_path container.metrics = json.dumps(metrics) self.session.commit() bot.info("[container][%s] %s" % (action, names['uri'])) return container