def build(self, repo, config=None, name=None, commit=None, tag="latest", recipe="Singularity", preview=False): '''trigger a build on Google Cloud (storage then compute) given a name recipe, and Github URI where the recipe can be found. Parameters ========== name: should be the complete uri that the user has requested to push. commit: a commit to use, not required, and can be parsed from URI repo: should correspond to a Github URL or (if undefined) used local repo. tag: a user specified tag, to take preference over tag in name config: The local config file to use. If the file doesn't exist, then we attempt looking up the config based on the name. recipe: If defined, limit builder to build a single recipe ''' bot.debug("BUILD %s" % repo) # Ensure that repo exists (200 response) if not self._healthy(repo): sys.exit(1) config = self._load_build_config(config) # If name not provided, parse name based on repository if name is None: name = '/'.join(repo.split('/')[-2:]) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name)) # First priority - user has provided a tag names['tag'] = tag or names['tag'] # If we still don't have custom tag, check the recipe if names['tag'] == "latest" and recipe != "Singularity": tag = get_recipe_tag(recipe) names = parse_image_name(remove_uri(name), tag=tag) # The commit is the version (after the @) commit = commit or names['version'] # Setup the build config = self._setup_build(name=names['url'], recipe=recipe, repo=repo, config=config, tag=tag, commit=commit) # The user only wants to preview the configuration if preview is True: return config # Otherwise, run the build! return self._run_build(config)
def push(self, path, name, tag=None): """push an image to Google Cloud Storage, meaning uploading it Parameters ========== path: should correspond to an absolte image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker """ path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.exit("%s does not exist." % path) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name), tag=tag) if names["version"] is None: version = get_file_hash(path, "sha256") names = parse_image_name(remove_uri(name), tag=tag, version=version) # Update metadata with names metadata = self.get_metadata(path, names=names) manifest = self._upload(source=path, destination=names["storage"], metadata=metadata) print(manifest["mediaLink"])
def push(self, path, name, tag=None): '''push an image to Google Cloud Storage, meaning uploading it path: should correspond to an absolte image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker ''' path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.error('%s does not exist.' % path) sys.exit(1) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name), tag=tag) if names['version'] is None: version = get_image_hash(path) names = parse_image_name(remove_uri(name), tag=tag, version=version) # Update metadata with names metadata = self.get_metadata(path, names=names) metadata = metadata['data'] metadata.update(names) manifest = self._upload(source=path, destination=names['storage'], metadata=metadata) print(manifest['mediaLink'])
def delete(self, image, force=False): """delete an image to Singularity Registry""" q = parse_image_name(remove_uri(image)) # If the registry is provided in the uri, use it if q["registry"] is None: q["registry"] = self.base # If the base doesn't start with http or https, add it q = self._add_https(q) url = "%s/container/%s/%s:%s" % ( q["registry"], q["collection"], q["image"], q["tag"], ) SREGISTRY_EVENT = self.authorize(request_type="delete", names=q) headers = {"Authorization": SREGISTRY_EVENT} self._update_headers(fields=headers) if confirm_delete(q["uri"], force) is True: response = self._delete(url) message = self._read_response(response) bot.info("Response %s, %s" % (response.status_code, message)) # add some error handling here?? else: bot.info("Delete cancelled.") return image
def remove(self, image, force=False): '''delete an image to Singularity Registry''' q = parse_image_name(remove_uri(image)) url = '%s/container/%s/%s:%s' % (self.base, q["collection"], q["image"], q["tag"]) SREGISTRY_EVENT = self.authorize(request_type="delete", names=q) headers = {'Authorization': SREGISTRY_EVENT } self._update_headers(fields=headers) continue_delete = True if force is False: response = input("Are you sure you want to delete %s?" % q['uri']) while len(response) < 1 or response[0].lower().strip() not in "ynyesno": response = input("Please answer yes or no: ") if response[0].lower().strip() in "no": continue_delete = False if continue_delete is True: response = self._delete(url) message = self._read_response(response) bot.info("Response %s, %s" %(response.status_code, message)) else: bot.info("Delete cancelled.")
def get(self, request, name): names = parse_image_name(name) container = get_container(names) # TODO: need to check permissions here # TODO: what to return when can't find container? if container is None: return Response(status=404) # Get other tags tags = [c.tag for c in container.collection.containers.all()] # Downloads path = "images/%s/%s:%s" % (names['collection'], names['image'], names['tag']) downloads = APIRequestCount.objects.filter( method="get", path__contains=path).count() data = { "deleted": False, # 2019-03-15T19:02:24.015Z "createdAt": container.add_date.strftime( '%Y-%m-%dT%H:%M:%S.%jZ'), # No idea what their format is... "hash": container.version, "size": container.metadata.get('size_mb'), "entityName": container.name, "collectionName": container.collection.name.split('/')[0], "containerName": container.name.split('/')[0], "tags": tags, "containerStars": container.collection.star_set.count(), "containerDownloads": downloads } return Response(data={"data": data}, status=200)
def get_request_collection(instance): """obtain the collection from a request Parameters ========== instance: should be an APIRequestLog object, with a response and path to parse """ import pickle pickle.dump(instance, open("instance.pkl", "wb")) from sregistry.utils import parse_image_name from shub.apps.main.models import Collection try: response = json.loads(instance.response) name = response["collection"] except: # Case 1: library endpoint if "/v1/images" in instance.path: collection_name = instance.path.replace("/v1/images/", "") # Case 2: shub endpoint else: collection_name = instance.path.replace("/api/container/", "") name = parse_image_name(collection_name)["collection"] try: collection = Collection.objects.get(name=name) except Collection.DoesNotExist: collection = None return collection
def container_query(q, across_collections=1): '''query for a container depending on the provided query string''' across_collections = bool(int(across_collections)) # Return complete lookup with image, collection, tag q = parse_image_name(q, defaults=False) if across_collections is True: if q['tag'] is not None: # Query across collections for image name and tag return Container.objects.filter( Q(name__contains=q['image']) | Q(tag__contains=q['tag'])).distinct() # Query across collections for image name return Container.objects.filter( Q(name__contains=q['image'])).distinct() # Query a particular collection for image name and tag if q['tag'] is not None: return Collection.objects.filter( Q(name__contains=q['image']) | Q(collection__name__contains=q['collection']) | Q(containers_tags__contains=q['tag'])).distinct() # Query a particular collection for image name return Collection.objects.filter( Q(name__contains=q['image']) | Q(collection__name__contains=q['collection'])).distinct()
def get(self, request, name): print("GET DownloadImageView") names = parse_image_name(name) container = get_container(names) # If no container, regardles of permissions, 404 if container is None: return Response(status=404) # Private containers we check the token if container.collection.private: token = get_token(request) # The user is not authenticated if not token: return Response(status=404) # Only owners and contributors can pull collection = container.collection if (token.user not in collection.owners.all() and token.user not in collection.contributors.all()): return Response(status=404) # Retrieve the url for minio storage = container.get_storage() url = minioExternalClient.presigned_get_object( MINIO_BUCKET, storage, expires=timedelta(minutes=MINIO_SIGNED_URL_EXPIRE_MINUTES), ) return redirect(url)
def push(self, path, name, tag=None): '''push an image to an S3 endpoint''' path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.exit('%s does not exist.' % path) # Extract the metadata names = parse_image_name(remove_uri(name), tag=tag) image_size = os.path.getsize(path) >> 20 # Create extra metadata, this is how we identify the image later # *important* bug in boto3 will return these capitalized # see https://github.com/boto/boto3/issues/1709 metadata = { 'sizemb': "%s" % image_size, 'client': 'sregistry', 'type': 'container' } ExtraArgs = {"Metadata": metadata} acl = self._get_and_update_setting('SREGISTRY_S3_OBJECT_ACL') if acl is not None: ExtraArgs['ACL'] = acl try: self.bucket.upload_file(path, names['storage'], ExtraArgs) except botocore.exceptions.ClientError as e: bot.exit( "Could not upload {} to bucket. Ensure you have sufficient permissions to put objects in the bucket (s3:PutObject), as well as modify the object ACL if SREGISTRY_S3_OBJECT_ACL is set (s3:PutObjectAcl): {}" .format(path, str(e)))
def push(self, path, name, tag=None): '''push an image to Singularity Registry path: should correspond to an absolte image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker ''' path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.error('%s does not exist.' %path) sys.exit(1) # This returns a data structure with collection, container, based on uri names = parse_image_name(remove_uri(name),tag=tag) # use Singularity client, if exists, to inspect to extract metadata metadata = self.get_metadata(path, names=names) # If you want a spinner bot.spinner.start() # do your push request here. Generally you want to except a KeyboardInterrupt # and give the user a status from the response bot.spinner.stop()
def get(self, name, quiet=False): '''Do a get for a container, and then a collection, and then return None if no result is found. Parameters ========== name: should coincide with either the collection name, or the container name with the collection. A query is done first for the collection, and then the container, and the path to the image file returned. ''' from sregistry.database.models import Collection, Container names = parse_image_name(remove_uri(name)) # First look for a collection (required) collection = self.get_collection(name=names['collection']) container = None if collection is not None: container = self.get_container(collection_id=collection.id, name=names['image'], tag=names['tag'], version=names['version']) if container is not None and quiet is False: # The container image file exists [local] if container.image is not None: print(container.image) # The container has a url (but not local file) elif container.url is not None: print(container.url) else: bot.info('No storage file found for %s' % name) return container
def get_metadata(self, image_file, names=None): '''extract metadata using Singularity inspect, if the executable is found. If not, return a reasonable default (the parsed image name) Parameters ========== image_file: the full path to a Singularity image names: optional, an extracted or otherwise created dictionary of variables for the image, likely from utils.parse_image_name ''' if names is None: names = {} metadata = {} # We can't return anything without image_file or names if image_file: if not os.path.exists(image_file): bot.error('Cannot find %s.' % image_file) return names or metadata # The user provided a file, but no names if not names: names = parse_image_name(remove_uri(image_file)) # Look for the Singularity Executable singularity = which('singularity')['message'] # Inspect the image, or return names only if os.path.exists(singularity) and image_file: from spython.main import Client as Singularity # Store the original quiet setting is_quiet = Singularity.quiet # We try and inspect, but not required (wont work within Docker) try: Singularity.quiet = True updates = Singularity.inspect(image=image_file) except: bot.warning( 'Inspect command not supported, metadata not included.') updates = None # Restore the original quiet setting Singularity.quiet = is_quiet # Try loading the metadata if updates is not None: try: updates = json.loads(updates) metadata.update(updates) except: pass metadata.update(names) return metadata
def pull(self, images, file_name=None, save=True, force=False, base=None, **kwargs): """pull an image from a docker hub. This is a (less than ideal) workaround that actually does the following: - creates a sandbox folder - adds docker layers, metadata folder, and custom metadata to it - converts to a squashfs image with build the docker manifests are stored with registry metadata. Parameters ========== images: refers to the uri given by the user to pull in the format <collection>/<namespace>. You should have an API that is able to retrieve a container based on parsing this uri. file_name: the user's requested name for the file. It can optionally be None if the user wants a default. save: if True, you should save the container to the database using self.add() base: the registry base, in case the client doesn't want to set in env. Returns ======= finished: a single container path, or list of paths """ if not isinstance(images, list): images = [images] bot.debug("Execution of PULL for %s images" % len(images)) # If used internally we want to return a list to the user. finished = [] for image in images: # 0. Update the base in case we aren't working with default base = self._update_base(image) q = parse_image_name(remove_uri(image), base=base) image_file = self._pull(file_name=file_name, save=save, force=force, names=q, kwargs=kwargs) finished.append(image_file) if len(finished) == 1: finished = finished[0] return finished
def get(self, request, name): names = parse_image_name(name) container = get_container(names) # TODO: need to check permissions here # TODO: what to return when can't find container? if container is None: return Response(status=404) return redirect(self.get_download_url(container))
def container_search(self, query, across_collections=False): """search for a specific container. If across collections is False, the query is parsed as a full container name and a specific container is returned. If across_collections is True, the container is searched for across collections. If across collections is True, details are not shown""" query = query.lower().strip("/") q = parse_image_name(remove_uri(query), defaults=False) if q["tag"] is not None: if across_collections is True: url = "%s/container/search/name/%s/tag/%s" % ( self.base, q["image"], q["tag"], ) else: url = "%s/container/search/collection/%s/name/%s/tag/%s" % ( self.base, q["collection"], q["image"], q["tag"], ) elif q["tag"] is None: if across_collections is True: url = "%s/container/search/name/%s" % (self.base, q["image"]) else: url = "%s/container/search/collection/%s/name/%s" % ( self.base, q["collection"], q["image"], ) result = self._get(url) if "containers" in result: result = result["containers"] if len(result) == 0: bot.info("No containers found.") sys.exit(1) bot.info("Containers %s" % query) rows = [] for c in result: rows.append(["%s/%s" % (c["collection"], c["name"]), c["tag"]]) bot.table(rows) return rows
def record(self, images, action='add'): '''record an image from an endpoint. This function is akin to a pull, but without retrieving the image. We only care about the list of images (uris) to look up, and then the action that the user wants to take Parameters ========== images: refers to the uri given by the user to pull in the format <collection>/<namespace>. You should have an API that is able to retrieve metadata for a container based on this url. action: the action to take with the record. By default we add it, meaning adding a record (metadata and file url) to the database. It is recommended to place the URL for the image download under the container.url field, and the metadata (the image manifest) should have a selfLink to indicate where it came from. ''' # Take a look at pull for an example of this logic. if not isinstance(images, list): images = [images] bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images))) for image in images: q = parse_image_name(remove_uri(image)) # Verify image existence, and obtain id url = "..." # This should be some url for your endpoint to get metadata bot.debug('Retrieving manifest at %s' % url) # Get the manifest, add a selfLink to it (good practice) manifest = self._get(url) manifest['selfLink'] = url # versions are very important! Since we aren't downloading the file, # If you don't have a version in your manifest, don't add it to the uri. # you will likely need to customize this string formation to make the # expected uri as in <collection>/<namespace>:<tag>@<version> if manifest['version'] is not None: image_uri = "%s/%s:%s@%s" % (manifest['collection'], manifest['name'], manifest['tag'], manifest['version']) else: image_uri = "%s/%s:%s" % (manifest['collection'], manifest['name'], manifest['tag']) # We again use the "add" function, but we don't give an image path # so it's just added as a record container = self.add(image_name=image_uri, metadata=manifest, url=manifest['image'])
def test_parse_image_name(): print("Testing utils.parse_image_name") from sregistry.utils import parse_image_name names = parse_image_name('ubuntu') ubuntu_check(names, 'ubuntu') names = parse_image_name('ubuntu:latest') ubuntu_check(names, 'ubuntu:latest') names = parse_image_name('library/ubuntu:latest') ubuntu_check(names, 'library/ubuntu:latest') # Version should include same with version names = parse_image_name('library/ubuntu:latest@version') assert names['version'] == 'version' assert names['uri'] == "library/ubuntu:latest@version" assert names['image'] == 'ubuntu' assert names['collection'] == 'library' assert names['registry'] is None assert names['tag'] == 'latest' assert names['url'] == "library/ubuntu" assert names['uri'] == 'library/ubuntu:latest@version' assert names['original'] == 'library/ubuntu:latest@version' assert names['storage'] == 'library/ubuntu:[email protected]'
def push(self, path, name, tag=None): '''push an image to your Storage. If the collection doesn't exist, it is created. Parameters ========== path: should correspond to an absolute image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker ''' path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.error('%s does not exist.' % path) sys.exit(1) # Parse image names names = parse_image_name(remove_uri(name), tag=tag) # Get the size of the file file_size = os.path.getsize(path) chunk_size = 4 * 1024 * 1024 storage_path = "/%s" % names['storage'] # Create / get the collection collection = self._get_or_create_collection(names['collection']) # The image name is the name followed by tag image_name = os.path.basename(names['storage']) # prepare the progress bar progress = 0 bot.show_progress(progress, file_size, length=35) # Put the (actual) container into the collection with open(path, 'rb') as F: self.conn.put_object(names['collection'], image_name, contents=F.read(), content_type='application/octet-stream') # Finish up bot.show_progress(iteration=file_size, total=file_size, length=35, carriage_return=True) # Newline to finish download sys.stdout.write('\n')
def get(self, request, name): names = parse_image_name(name) container = get_container(names) # TODO: need to check permissions here # TODO: what to return when can't find container? if container is None: return Response(status=404) # Private containers are simply not accessible - no way to authenticate if container.collection.private: return Response(status=404) return redirect(self.get_download_url(container))
def test_parse_image_name(): print("Testing utils.parse_image_name") from sregistry.utils import parse_image_name names = parse_image_name("ubuntu") ubuntu_check(names, "ubuntu") names = parse_image_name("ubuntu:latest") ubuntu_check(names, "ubuntu:latest") names = parse_image_name("library/ubuntu:latest") ubuntu_check(names, "library/ubuntu:latest") # Version should include same with version names = parse_image_name("library/ubuntu:latest@version") assert names["version"] == "version" assert names["uri"] == "library/ubuntu:latest@version" assert names["image"] == "ubuntu" assert names["collection"] == "library" assert names["registry"] is None assert names["tag"] == "latest" assert names["url"] == "library/ubuntu" assert names["uri"] == "library/ubuntu:latest@version" assert names["original"] == "library/ubuntu:latest@version" assert names["storage"] == "library/ubuntu:[email protected]"
def record(self, images, action='add'): '''record an image from an endpoint. This function is akin to a pull, but without retrieving the image. We only care about the list of images (uris) to look up, and then the action that the user wants to take Parameters ========== images: refers to the uri given by the user to pull in the format <collection>/<namespace>. You should have an API that is able to retrieve metadata for a container based on this url. action: the action to take with the record. By default we add it, meaning adding a record (metadata and file url) to the database. It is recommended to place the URL for the image download under the container.url field, and the metadata (the image manifest) should have a selfLink to indicate where it came from. ''' # Take a look at pull for an example of this logic. if not isinstance(images, list): images = [images] bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images))) for image in images: q = parse_image_name(remove_uri(image)) # Use container search to find the container based on uri bot.info('Searching for %s in gs://%s' % (q['uri'], self._bucket_name)) matches = self._container_query(q['uri'], quiet=True) if len(matches) == 0: bot.info('No matching containers found.') sys.exit(0) # We give the first match, the uri should be unique and known image = matches[0] image_uri = q['uri'] if "uri" in image.metadata: image_uri = image.metadata['uri'] # Update metadata with selfLink metadata = image.metadata metadata['selfLink'] = image.self_link # Use add without image path so added as a record container = self.add(image_uri=image_uri, metadata=metadata, url=image.media_link)
def get(self, request, name): # The request specifies ?arch=amd64 but that's all we got print("GET GetImageView") names = parse_image_name(name) # If an arch is not specified, redirect to push view arch = request.query_params.get("arch", "amd64") container = get_container(names) # If an arch is defined, ensure it matches the request arch = "amd64" if arch and container is not None: if container.metadata.get("arch", "amd64") != "amd64": return Response(status=404) # If no container, regardles of permissions, 404 if container is None: return Response(status=404) # Private containers we check the token if container.collection.private: token = get_token(request) # Only owners and contributors can pull collection = container.collection if (token.user not in collection.owners.all() and token.user not in collection.contributors.all()): return Response(status=404) # Generate log for downloads (async with worker) django_rq.enqueue( generate_log, view_name="shub.apps.api.urls.containers.ContainerDetailByName", ipaddr=request.META.get("HTTP_X_FORWARDED_FOR", None), method=request.method, params=request.query_params.dict(), request_path=request.path, remote_addr=request.META.get("REMOTE_ADDR", ""), host=request.get_host(), request_data=request.data, auth_header=request.META.get("HTTP_AUTHORIZATION"), ) data = generate_container_metadata(container) return Response(data={"data": data}, status=200)
def check_tasks(client, endpoint_id): ''' check on task list and compare against local, update with any containers that have finished status ''' from sregistry.utils import parse_image_name events = [] for task in client.transfer_client.task_list(): tid = task.data['task_id'] # Tasks still processing won't have events available if task.data['status'] in ["SUCCEEDED", "FAILED"]: for event in client.transfer_client.task_successful_transfers(tid): # Link to task history link = "https://globus.org/app/activity/%s" %tid dest = event['destination_path'] source = event['source_path'] # Case 1: Add images transferred here to sregistry if task.data['destination_endpoint_id'] == endpoint_id: # Do we have the container? if os.path.exists(dest): # Parse name from source_path metadata = {'globus_event': event, 'globus_task_id': tid, 'selfLink': link } names = parse_image_name(os.path.basename(source)) metadata.update(names) result = client.add(image_uri=names['uri'], image_path=dest, metadata=metadata) events.append(names['uri']) # Case 2: remove temporary files for finished remote transfers else: if os.path.exists(event['source_path']): os.remove(event['source_path']) return events
def get_metadata(self, image_file, names={}): '''extract metadata using Singularity inspect, if the executable is found. If not, return a reasonable default (the parsed image name) Parameters ========== image_file: the full path to a Singularity image names: optional, an extracted or otherwise created dictionary of variables for the image, likely from utils.parse_image_name ''' metadata = dict() # We can't return anything without image_file or names if image_file is not None: if not os.path.exists(image_file): bot.error('Cannot find %s.' % image_file) return names # The user provided a file, but no names if not names: names = parse_image_name(remove_uri(image_file)) # Look for the Singularity Executable singularity = which('singularity')['message'] # Inspect the image, or return names only if os.path.exists(singularity) and image_file is not None: from sregistry.client import Singularity cli = Singularity() updates = cli.inspect(image_path=image_file, quiet=True) # Try loading the metadata if updates is not None: try: updates = json.loads(updates) metadata.update(updates) except: pass metadata.update(names) return metadata
def rename(self, image_name, path): '''rename performs a move, but ensures the path is maintained in storage Parameters ========== image_name: the image name (uri) to rename to. path: the name to rename (basename is taken) ''' container = self.get(image_name, quiet=True) if container is not None: if container.image is not None: # The original directory for the container stays the same dirname = os.path.dirname(container.image) # But we derive a new filename and uri names = parse_image_name(remove_uri(path)) storage = os.path.join(self.storage, os.path.dirname(names['storage'])) # This is the collection folder if not os.path.exists(storage): os.mkdir(storage) # Here we get the new full path, rename the container file fullpath = os.path.abspath(os.path.join(dirname, names['storage'])) container = self.cp(move_to=fullpath, container=container, command="rename") # On successful rename of file, update the uri if container is not None: container.uri = names['uri'] self.session.commit() return container bot.warning('%s not found' % (image_name))
def get(self, request, name): print("GET DownloadImageView") names = parse_image_name(name) container = get_container(names) # If no container, regardles of permissions, 404 if container is None: return Response(status=404) # Private containers we check the token if container.collection.private: token = get_token(request) # Only owners and contributors can pull collection = container.collection if (token.user not in collection.owners.all() and token.user not in collection.contributors.all()): return Response(status=404) return redirect(self.get_download_url(container))
def record(self, images, action='add'): '''record an image from an endpoint. This function is akin to a pull, but without retrieving the image. We only care about the list of images (uris) to look up, and then the action that the user wants to take Parameters ========== images: refers to the uri given by the user to pull in the format <collection>/<namespace>. You should have an API that is able to retrieve metadata for a container based on this url. action: the action to take with the record. By default we add it, meaning adding a record (metadata and file url) to the database. It is recommended to place the URL for the image download under the container.url field, and the metadata (the image manifest) should have a selfLink to indicate where it came from. ''' # Take a look at pull for an example of this logic. if not isinstance(images, list): images = [images] bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images))) for image in images: names = parse_image_name(remove_uri(image)) # Dropbox path is the path in storage with a slash dropbox_path = '/%s' % names['storage'] # First ensure that exists if self.exists(dropbox_path) is True: # Get metadata from dropbox, and then update with sregistry metadata = self.dbx.files_get_metadata(dropbox_path) metadata = self._get_metadata(dbx_metadata=metadata) # Add image as a record container = self.add(image_uri=dropbox_path.strip('/'), metadata=metadata, url=metadata['path_lower'])
def rename(self, image_name, path): '''rename performs a move, but ensures the path is maintained in storage Parameters ========== image_name: the image name (uri) to rename to. path: the name to rename (basename is taken) ''' container = self.get(image_name, quiet=True) if container: if container.image: # Derive a new filename and url in storage names = parse_image_name(remove_uri(path), version=container.version) storage = self._get_storage_name(names) dirname = os.path.dirname(storage) # This is the collection folder if not os.path.exists(dirname): os.makedirs(dirname) container = self.cp(move_to=storage, container=container, command="rename") # On successful rename of file, update the uri if container is not None: # Create the collection if doesn't exist collection = self.get_or_create_collection(names['collection']) self.session.commit() # Then update the container container = update_container_metadata(container, collection, names) self.session.commit() return container bot.warning('%s not found' % image_name)
def record(self, images, action='add'): '''record an image from an endpoint. This function is akin to a pull, but without retrieving the image. We only care about the list of images (uris) to look up, and then the action that the user wants to take Parameters ========== images: refers to the uri given by the user to pull in the format <collection>/<namespace>. You should have an API that is able to retrieve metadata for a container based on this url. action: the action to take with the record. By default we add it, meaning adding a record (metadata and file url) to the database. It is recommended to place the URL for the image download under the container.url field, and the metadata (the image manifest) should have a selfLink to indicate where it came from. ''' # Take a look at pull for an example of this logic. if not isinstance(images, list): images = [images] bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images))) # If used internally we want to return a list to the user. for image in images: # 0. Update the base in case we aren't working with default base = self._update_base(image) q = parse_image_name(remove_uri(image), base=base) digest = q['version'] or q['tag'] # This is the Docker Hub namespace and repository manifests = self._get_manifests(q['url'], digest) # This is the url where the manifests were obtained url = self._get_manifest_selfLink(q['url'], digest) # We again use the "add" function, but we don't give an image path # so it's just added as a record container = self.add(image_uri=q['uri'], metadata=manifests, url=url)