def push(self, path, name, tag=None): '''push an image to Google Cloud Storage, meaning uploading it path: should correspond to an absolte image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker ''' path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.error('%s does not exist.' % path) sys.exit(1) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name), tag=tag) if names['version'] is None: version = get_image_hash(path) names = parse_image_name(remove_uri(name), tag=tag, version=version) # Update metadata with names metadata = self.get_metadata(path, names=names) metadata = metadata['data'] metadata.update(names) manifest = self._upload(source=path, destination=names['storage'], metadata=metadata) print(manifest['mediaLink'])
def add(self, image_path=None, image_uri=None, image_name=None, url=None, metadata=None, save=True, copy=False): '''get or create a container, including the collection to add it to. This function can be used from a file on the local system, or via a URL that has been downloaded. Either way, if one of url, version, or image_file is not provided, the model is created without it. If a version is not provided but a file path is, then the file hash is used. Parameters ========== image_path: full path to image file image_name: if defined, the user wants a custom name (and not based on uri) metadata: any extra metadata to keep for the image (dict) save: if True, move the image to the cache if it's not there copy: If True, copy the image instead of moving it. image_name: a uri that gets parsed into a names object that looks like: {'collection': 'vsoch', 'image': 'hello-world', 'storage': 'vsoch/hello-world-latest.img', 'tag': 'latest', 'version': '12345' 'uri': 'vsoch/hello-world:latest@12345'} After running add, the user will take some image in a working directory, add it to the database, and have it available for search and use under SREGISTRY_STORAGE/<collection>/<container> If the container was retrieved from a webby place, it should have version If no version is found, the file hash is used. ''' from sregistry.database.models import (Container, Collection) # We can only save if the image is provided if image_path is not None: if not os.path.exists(image_path) and save is True: bot.error('Cannot find %s' % image_path) sys.exit(1) # An image uri is required for version, tag, etc. if image_uri is None: bot.error('You must provide an image uri <collection>/<namespace>') sys.exit(1) names = parse_image_name(remove_uri(image_uri)) bot.debug('Adding %s to registry' % names['uri']) # If Singularity is installed, inspect image for metadata metadata = self.get_metadata(image_path, names=names) collection = self.get_or_create_collection(names['collection']) # If save, move to registry storage first if save is True and image_path is not None: # If the user hasn't defined a custom name if image_name is None: image_name = self._get_storage_name(names) if copy is True: copyfile(image_path, image_name) else: shutil.move(image_path, image_name) image_path = image_name # Get a hash of the file for the version, or use provided version = names.get('version') if version is None: if image_path is not None: version = get_image_hash(image_path) else: version = '' # we can't determine a version, not in API/no file names['version'] = version # Just in case the client didn't provide it, see if we have in metadata if url is None and "url" in metadata: url = metadata['url'] # First check that we don't have one already! container = self.get_container(name=names['image'], collection_id=collection.id, tag=names['tag'], version=version) # The container did not exist, create it if container is None: action = "new" container = Container(metrics=json.dumps(metadata), name=names['image'], image=image_path, client=self.client_name, tag=names['tag'], version=version, url=url, uri=names['uri'], collection_id=collection.id) self.session.add(container) collection.containers.append(container) # The container existed, update it. else: action = "update" metrics = json.loads(container.metrics) metrics.update(metadata) container.url = url container.client = self.client_name if image_path is not None: container.image = image_path container.metrics = json.dumps(metrics) self.session.commit() bot.info("[container][%s] %s" % (action, names['uri'])) return container
def push(self, path, name, tag=None): '''push an image to Google Cloud Drive, meaning uploading it path: should correspond to an absolte image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker ''' # The root of the drive for containers (the parent folder) parent = self._get_or_create_folder(self._base) image = None path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.error('%s does not exist.' % path) sys.exit(1) names = parse_image_name(remove_uri(name), tag=tag) if names['version'] is None: version = get_image_hash(path) names = parse_image_name(remove_uri(name), tag=tag, version=version) # Update metadata with names, flatten to only include labels metadata = self.get_metadata(path, names=names) metadata = metadata['data'] metadata.update(names) metadata.update(metadata['attributes']['labels']) del metadata['attributes'] file_metadata = { 'name': names['storage'], 'mimeType': 'application/octet-stream', 'parents': [parent['id']], 'properties': metadata } media = MediaFileUpload(path, resumable=True) try: bot.spinner.start() image = self._service.files().create(body=file_metadata, media_body=media, fields='id').execute() # Add a thumbnail! thumbnail = get_thumbnail() with open(thumbnail, "rb") as f: body = { "contentHints": { "thumbnail": { "image": base64.urlsafe_b64encode(f.read()).decode('utf8'), "mimeType": "image/png" } } } image = self._service.files().update(fileId=image['id'], body=body).execute() bot.spinner.stop() print(image['name']) except HttpError: bot.error('Error uploading %s' % path) pass return image