def container_query(self, query):
'''search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
'''
results = set()
query = remove_uri(query)
# Here we get names of collections, and then look up containers
for container in self.conn.get_account()[1]:
# The result here is just the name
for result in self.conn.get_container(container['name'])[1]:
if query in collection['name']:
results.add('%s/%s' % (container['name'], result['name']))
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table([list(results)])
return list(results)
def get(self, name, quiet=False):
'''Do a get for a container, and then a collection, and then return None
if no result is found.
Parameters
==========
name: should coincide with either the collection name, or the container
name with the collection. A query is done first for the collection,
and then the container, and the path to the image file returned.
'''
from sregistry.database.models import Collection, Container
names = parse_image_name(remove_uri(name))
# First look for a collection (required)
collection = self.get_collection(name=names['collection'])
container = None
if collection is not None:
container = self.get_container(collection_id=collection.id,
name=names['image'],
tag=names['tag'],
version=names['version'])
if container is not None and quiet is False:
# The container image file exists [local]
if container.image is not None:
print(container.image)
# The container has a url (but not local file)
elif container.url is not None:
print(container.url)
else:
bot.info('No remote url or storage file found for %s' % name)
return container
def push(self, path, name, tag=None):
"""push an image to an S3 endpoint"""
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.exit("%s does not exist." % path)
# Extract the metadata
names = parse_image_name(remove_uri(name), tag=tag)
image_size = os.path.getsize(path) >> 20
# Create extra metadata, this is how we identify the image later
# *important* bug in boto3 will return these capitalized
# see https://github.com/boto/boto3/issues/1709
metadata = {
"sizemb": "%s" % image_size,
"client": "sregistry",
"type": "container"
}
ExtraArgs = {"Metadata": metadata}
acl = self._get_and_update_setting("SREGISTRY_S3_OBJECT_ACL")
if acl is not None:
ExtraArgs["ACL"] = acl
try:
self.bucket.upload_file(path, names["storage"], ExtraArgs)
except botocore.exceptions.ClientError as e:
bot.exit(
"Could not upload {} to bucket. Ensure you have sufficient permissions to put objects in the bucket (s3:PutObject), as well as modify the object ACL if SREGISTRY_S3_OBJECT_ACL is set (s3:PutObjectAcl): {}"
.format(path, str(e)))
def container_query(self, query):
'''search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
'''
results = []
query = remove_uri(query)
# Parse through folders (collections):
for entry in self.dbx.files_list_folder('').entries:
# Parse through containers
for item in self.dbx.files_list_folder(entry.path_lower).entries:
name = item.name.replace('.simg', '')
name = "%s/%s" % (entry.name, name)
if query in name:
results.append([name])
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table(results)
return results
def get_metadata(self, image_file, names=None):
'''extract metadata using Singularity inspect, if the executable is found.
If not, return a reasonable default (the parsed image name)
Parameters
==========
image_file: the full path to a Singularity image
names: optional, an extracted or otherwise created dictionary of
variables for the image, likely from utils.parse_image_name
'''
if names is None:
names = {}
metadata = {}
# We can't return anything without image_file or names
if image_file:
if not os.path.exists(image_file):
bot.error('Cannot find %s.' % image_file)
return names or metadata
# The user provided a file, but no names
if not names:
names = parse_image_name(remove_uri(image_file))
# Look for the Singularity Executable
singularity = which('singularity')['message']
# Inspect the image, or return names only
if os.path.exists(singularity) and image_file:
from spython.main import Client as Singularity
# Store the original quiet setting
is_quiet = Singularity.quiet
# We try and inspect, but not required (wont work within Docker)
try:
Singularity.quiet = True
updates = Singularity.inspect(image=image_file)
except:
bot.warning(
'Inspect command not supported, metadata not included.')
updates = None
# Restore the original quiet setting
Singularity.quiet = is_quiet
# Try loading the metadata
if updates is not None:
try:
updates = json.loads(updates)
metadata.update(updates)
except:
pass
metadata.update(names)
return metadata
def pull(self,
images,
file_name=None,
save=True,
force=False,
base=None,
**kwargs):
"""pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
base: the registry base, in case the client doesn't want to set in env.
Returns
=======
finished: a single container path, or list of paths
"""
if not isinstance(images, list):
images = [images]
bot.debug("Execution of PULL for %s images" % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
# 0. Update the base in case we aren't working with default
base = self._update_base(image)
q = parse_image_name(remove_uri(image), base=base)
image_file = self._pull(file_name=file_name,
save=save,
force=force,
names=q,
kwargs=kwargs)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def container_search(self, query, across_collections=False):
"""search for a specific container. If across collections is False,
the query is parsed as a full container name and a specific container
is returned. If across_collections is True, the container is searched
for across collections. If across collections is True, details are
not shown"""
query = query.lower().strip("/")
q = parse_image_name(remove_uri(query), defaults=False)
if q["tag"] is not None:
if across_collections is True:
url = "%s/container/search/name/%s/tag/%s" % (
self.base,
q["image"],
q["tag"],
)
else:
url = "%s/container/search/collection/%s/name/%s/tag/%s" % (
self.base,
q["collection"],
q["image"],
q["tag"],
)
elif q["tag"] is None:
if across_collections is True:
url = "%s/container/search/name/%s" % (self.base, q["image"])
else:
url = "%s/container/search/collection/%s/name/%s" % (
self.base,
q["collection"],
q["image"],
)
result = self._get(url)
if "containers" in result:
result = result["containers"]
if len(result) == 0:
bot.info("No containers found.")
sys.exit(1)
bot.info("Containers %s" % query)
rows = []
for c in result:
rows.append(["%s/%s" % (c["collection"], c["name"]), c["tag"]])
bot.table(rows)
return rows
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
for image in images:
q = parse_image_name(remove_uri(image))
# Verify image existence, and obtain id
url = "..." # This should be some url for your endpoint to get metadata
bot.debug('Retrieving manifest at %s' % url)
# Get the manifest, add a selfLink to it (good practice)
manifest = self._get(url)
manifest['selfLink'] = url
# versions are very important! Since we aren't downloading the file,
# If you don't have a version in your manifest, don't add it to the uri.
# you will likely need to customize this string formation to make the
# expected uri as in <collection>/<namespace>:<tag>@<version>
if manifest['version'] is not None:
image_uri = "%s/%s:%s@%s" % (manifest['collection'],
manifest['name'], manifest['tag'],
manifest['version'])
else:
image_uri = "%s/%s:%s" % (manifest['collection'], manifest['name'],
manifest['tag'])
# We again use the "add" function, but we don't give an image path
# so it's just added as a record
container = self.add(image_name=image_uri,
metadata=manifest,
url=manifest['image'])
def push(self, path, name, tag=None):
'''push an image to your Storage. If the collection doesn't exist,
it is created.
Parameters
==========
path: should correspond to an absolute image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
'''
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' % path)
sys.exit(1)
# Parse image names
names = parse_image_name(remove_uri(name), tag=tag)
# Get the size of the file
file_size = os.path.getsize(path)
chunk_size = 4 * 1024 * 1024
storage_path = "/%s" % names['storage']
# Create / get the collection
collection = self._get_or_create_collection(names['collection'])
# The image name is the name followed by tag
image_name = os.path.basename(names['storage'])
# prepare the progress bar
progress = 0
bot.show_progress(progress, file_size, length=35)
# Put the (actual) container into the collection
with open(path, 'rb') as F:
self.conn.put_object(names['collection'],
image_name,
contents=F.read(),
content_type='application/octet-stream')
# Finish up
bot.show_progress(iteration=file_size,
total=file_size,
length=35,
carriage_return=True)
# Newline to finish download
sys.stdout.write('\n')
def push(self, path, name, tag=None):
'''push an image to Google Cloud Storage, meaning uploading it
path: should correspond to an absolte image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
'''
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' % path)
sys.exit(1)
# This returns a data structure with collection, container, based on uri
names = parse_image_name(remove_uri(name), tag=tag)
if names['version'] is None:
version = get_image_hash(path)
names = parse_image_name(remove_uri(name), tag=tag, version=version)
# Update metadata with names
metadata = self.get_metadata(path, names=names)
metadata = metadata['data']
metadata.update(names)
#TODO: use multiple threads/other to upload
manifest = self._upload(source=path,
destination=names['storage'],
metadata=metadata)
# If result is successful, save container record
if manifest is not None:
metadata.update(manifest)
container = self.add(image_uri=names['uri'],
metadata=metadata,
url=manifest['mediaLink'])
print(manifest['mediaLink'])
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
for image in images:
q = parse_image_name(remove_uri(image))
# Use container search to find the container based on uri
bot.info('Searching for %s in gs://%s' % (q['uri'], self._bucket_name))
matches = self._container_query(q['uri'], quiet=True)
if len(matches) == 0:
bot.info('No matching containers found.')
sys.exit(0)
# We give the first match, the uri should be unique and known
image = matches[0]
image_uri = q['uri']
if "uri" in image.metadata:
image_uri = image.metadata['uri']
# Update metadata with selfLink
metadata = image.metadata
metadata['selfLink'] = image.self_link
# Use add without image path so added as a record
container = self.add(image_uri=image_uri,
metadata=metadata,
url=image.media_link)
def main(args, parser, extra):
from sregistry.main import get_client
for query in args.query:
original = query
query = remove_uri(query)
if query in ['', '*']:
query = None
try:
cli = get_client(original, quiet=args.quiet)
cli.announce(args.command)
cli.search(query, args=args)
except NotImplementedError:
msg = "search is not implemented for %s. Why don't you add it?"
bot.exit(msg % cli.client_name)
def rename(self, image_name, path):
'''rename performs a move, but ensures the path is maintained in storage
Parameters
==========
image_name: the image name (uri) to rename to.
path: the name to rename (basename is taken)
'''
container = self.get(image_name, quiet=True)
if container is not None:
if container.image is not None:
# The original directory for the container stays the same
dirname = os.path.dirname(container.image)
# But we derive a new filename and uri
names = parse_image_name(remove_uri(path))
storage = os.path.join(self.storage,
os.path.dirname(names['storage']))
# This is the collection folder
if not os.path.exists(storage):
os.mkdir(storage)
# Here we get the new full path, rename the container file
fullpath = os.path.abspath(os.path.join(dirname, names['storage']))
container = self.cp(move_to=fullpath,
container=container,
command="rename")
# On successful rename of file, update the uri
if container is not None:
container.uri = names['uri']
self.session.commit()
return container
bot.warning('%s not found' % (image_name))
def get_metadata(self, image_file, names={}):
'''extract metadata using Singularity inspect, if the executable is found.
If not, return a reasonable default (the parsed image name)
Parameters
==========
image_file: the full path to a Singularity image
names: optional, an extracted or otherwise created dictionary of
variables for the image, likely from utils.parse_image_name
'''
metadata = dict()
# We can't return anything without image_file or names
if image_file is not None:
if not os.path.exists(image_file):
bot.error('Cannot find %s.' % image_file)
return names
# The user provided a file, but no names
if not names:
names = parse_image_name(remove_uri(image_file))
# Look for the Singularity Executable
singularity = which('singularity')['message']
# Inspect the image, or return names only
if os.path.exists(singularity) and image_file is not None:
from sregistry.client import Singularity
cli = Singularity()
updates = cli.inspect(image_path=image_file, quiet=True)
# Try loading the metadata
if updates is not None:
try:
updates = json.loads(updates)
metadata.update(updates)
except:
pass
metadata.update(names)
return metadata
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
# If used internally we want to return a list to the user.
for image in images:
# 0. Update the base in case we aren't working with default
base = self._update_base(image)
q = parse_image_name(remove_uri(image), base=base)
digest = q['version'] or q['tag']
# This is the Docker Hub namespace and repository
manifests = self._get_manifests(q['url'], digest)
# This is the url where the manifests were obtained
url = self._get_manifest_selfLink(q['url'], digest)
# We again use the "add" function, but we don't give an image path
# so it's just added as a record
container = self.add(image_uri=q['uri'], metadata=manifests, url=url)
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
for image in images:
names = parse_image_name(remove_uri(image))
# Dropbox path is the path in storage with a slash
dropbox_path = '/%s' % names['storage']
# First ensure that exists
if self.exists(dropbox_path) is True:
# Get metadata from dropbox, and then update with sregistry
metadata = self.dbx.files_get_metadata(dropbox_path)
metadata = self._get_metadata(dbx_metadata=metadata)
# Add image as a record
container = self.add(image_uri=dropbox_path.strip('/'),
metadata=metadata,
url=metadata['path_lower'])
def rename(self, image_name, path):
'''rename performs a move, but ensures the path is maintained in storage
Parameters
==========
image_name: the image name (uri) to rename to.
path: the name to rename (basename is taken)
'''
container = self.get(image_name, quiet=True)
if container:
if container.image:
# Derive a new filename and url in storage
names = parse_image_name(remove_uri(path),
version=container.version)
storage = self._get_storage_name(names)
dirname = os.path.dirname(storage)
# This is the collection folder
if not os.path.exists(dirname):
os.makedirs(dirname)
container = self.cp(move_to=storage,
container=container,
command="rename")
# On successful rename of file, update the uri
if container is not None:
# Create the collection if doesn't exist
collection = self.get_or_create_collection(names['collection'])
self.session.commit()
# Then update the container
container = update_container_metadata(container, collection,
names)
self.session.commit()
return container
bot.warning('%s not found' % image_name)
def search_collection(self, query):
'''collection search will list all containers for a specific
collection. We assume query is the name of a collection'''
query = query.lower().strip('/')
q = parse_image_name(remove_uri(query), defaults=False)
# Workaround for now - the Singularity Hub search endpoind needs fixing
containers = self.list(quiet=True)
rows = []
for result in containers:
if re.search(query, result[1]):
rows.append(result)
if len(rows) > 0:
bot.table(rows)
else:
bot.info('No containers found.')
return rows
def delete(self, image, force=False):
"""delete an image from an S3 bucket"""
q = parse_image_name(remove_uri(image))
uri = q["storage"]
try:
_object = self.bucket.Object(uri)
_object.load(
) # this throws an exception if the object does not exist! -> if delete() fails no exception is thrown...
if confirm_delete(uri, force) is True:
_object.delete()
else:
bot.info("Delete cancelled.")
except Exception as e: # pylint: disable=broad-except
bot.error("Could not delete object {}: {}".format(uri, str(e)))
return None
return image
def push(self, path, name, tag=None):
'''push an image to an S3 endpoint'''
path = os.path.abspath(path)
image = os.path.basename(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' % path)
sys.exit(1)
# Extract the metadata
names = parse_image_name(remove_uri(name), tag=tag)
image_size = os.path.getsize(path) >> 20
# Create extra metadata, this is how we identify the image later
# *important* bug in boto3 will return these capitalized
# see https://github.com/boto/boto3/issues/1709
metadata = {'sizemb': "%s" % image_size, 'client': 'sregistry'}
self.bucket.upload_file(path, names['storage_uri'], {"Metadata": metadata})
def add(
self,
image_path=None,
image_uri=None,
image_name=None,
url=None,
metadata=None,
save=True,
copy=False,
):
"""dummy add simple returns an object that mimics a database entry, so the
calling function (in push or pull) can interact with it equally. Most
variables (other than image_path) are not used."""
# We can only save if the image is provided
if image_path is not None:
if not os.path.exists(image_path):
bot.exit("Cannot find %s" % image_path)
if image_uri is None:
bot.exit("You must provide an image uri <collection>/<namespace>")
names = parse_image_name(remove_uri(image_uri))
bot.debug("Added %s to filesystem" % names["uri"])
# Create a dummy container on the fly
class DummyContainer:
def __init__(self, image_path, client_name, url, names):
self.image = image_path
self.client = client_name
self.url = url
self.name = names["image"]
self.tag = names["tag"]
self.uri = names["uri"]
container = DummyContainer(image_path, self.client_name, url, names)
bot.info("[container][add] %s" % names["uri"])
return container
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
if not isinstance(images,list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' %(action, len(images)))
for image in images:
q = parse_image_name(remove_uri(image))
# Verify image existence, and obtain id
url = "%s/container/%s/%s:%s" %(self.base, q['collection'], q['image'], q['tag'])
bot.debug('Retrieving manifest at %s' %url)
# Get the manifest, add a selfLink to it
manifest = self._get(url)
manifest['selfLink'] = url
image_uri = "%s:%s@%s" %(manifest['name'], manifest['tag'], manifest['version'])
container = self.add(image_uri=image_uri,
metadata=manifest,
url=manifest['image'])
def share(self, query, share_to=None):
'''share will use the client to get a shareable link for an image of choice.
the functions returns a url of choice to send to a recipient.
'''
names = parse_image_name(remove_uri(query))
# Dropbox path is the path in storage with a slash
dropbox_path = '/%s' % names['storage']
# First ensure that exists
if self.exists(dropbox_path) is True:
# Create new shared link
try:
share = self.dbx.sharing_create_shared_link_with_settings(dropbox_path)
# Already exists!
except ApiError as err:
share = self.dbx.sharing_create_shared_link(dropbox_path)
bot.info(share.url)
return share.url
def get(self, name, quiet=False):
"""Do a get for a container, and then a collection, and then return None
if no result is found.
Parameters
==========
name: should coincide with either the collection name, or the container
name with the collection. A query is done first for the collection,
and then the container, and the path to the image file returned.
"""
names = parse_image_name(remove_uri(name))
# First look for a collection (required)
collection = self.get_collection(name=names["collection"])
container = None
if collection:
container = self.get_container(
collection_id=collection.id,
name=names["image"],
tag=names["tag"],
version=names["version"],
)
if container and not quiet:
# The container image file exists [local]
if container.image:
print(container.image)
# The container has a url (but not local file)
elif container.url:
print(container.url)
else:
bot.info("No storage file found for %s" % name)
return container
def pull(self, images, file_name=None, save=True, **kwargs):
"""pull an image from a dropbox. The image is found based on the storage
uri
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
force = False
if "force" in kwargs:
force = kwargs["force"]
if not isinstance(images, list):
images = [images]
bot.debug("Execution of PULL for %s images" % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
names = parse_image_name(remove_uri(image))
# Dropbox path is the path in storage with a slash
dropbox_path = "/%s" % names["storage"]
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = self._get_storage_name(names)
# If the file already exists and force is False
if os.path.exists(file_name) and force is False:
bot.exit("Image exists! Remove first, or use --force to overwrite")
# First ensure that exists
if self.exists(dropbox_path) is True:
# _stream is a function to stream using the response to start
metadata, response = self.dbx.files_download(dropbox_path)
image_file = self._stream(response, stream_to=file_name)
# parse the metadata (and add inspected image)
metadata = self._get_metadata(image_file, metadata)
# If we save to storage, the uri is the dropbox_path
if save is True:
container = self.add(
image_path=image_file,
image_uri=dropbox_path.strip("/"),
metadata=metadata,
url=response.url,
)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
else:
bot.error(
"%s does not exist. Try sregistry search to see images." %
dropbox_path)
if len(finished) == 1:
finished = finished[0]
return finished
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from google drive, based on a query (uri or id)
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
q = parse_image_name(remove_uri(image))
# Use container search to find the container based on uri
bot.info('Searching for %s in drive://%s' % (q['uri'], self._base))
matches = self._container_query(q['uri'], quiet=True)
if len(matches) == 0:
bot.info('No matching containers found.')
sys.exit(0)
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = q['storage'].replace('/', '-')
# We give the first match, the uri should be unique and known
image = matches[0]
request = self._service.files().get_media(fileId=image['id'])
with open(file_name, 'wb') as fh:
downloader = MediaIoBaseDownload(fh, request)
done = False
bar = None
# Download and update the user with progress bar
while done is False:
status, done = downloader.next_chunk()
response = None
# Create bar on first call
if bar is None:
total = status.total_size / (1024 * 1024.0)
bar = ProgressBar(expected_size=total, filled_char='=')
bar.show(status.resumable_progress / (1024 * 1024.0))
# If the user is saving to local storage, you need to assumble the uri
# here in the expected format <collection>/<namespace>:<tag>@<version>
if save is True:
image_uri = q['uri']
if "uri" in image:
image_uri = image['uri']
# Update metadata with selfLink
image['selfLink'] = downloader._uri
container = self.add(image_path=image_file,
image_uri=image_uri,
metadata=image,
url=downloader._uri)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def build(self, recipe, name, extra=None):
"""push a recipe file to Singularity Registry server for a Google
Cloud (or similar) build
"""
path = os.path.abspath(recipe)
bot.debug("BUILD %s" % path)
if extra is None:
extra = {}
if not os.path.exists(path):
bot.exit("%s does not exist." % path)
if not os.path.isfile(path):
bot.exit("Build takes a Singularity recipe file")
valid = ["google_build"]
# Only one valid type
if "google_build" not in extra:
bot.exit(
"Please include --builder google_build as the last extra arugment for Google Cloud Build"
)
builder_type = None
for builder_type in extra:
if builder_type in valid:
break
# Must have valid builder type
if builder_type is None:
bot.exit("Invalid builder type.")
# Extract the metadata
names = parse_image_name(remove_uri(name))
# COLLECTION ###################################################################
# If the registry is provided in the uri, use it
if names["registry"] is None:
names["registry"] = self.base
# If the base doesn't start with http or https, add it
names = self._add_https(names)
# Recipe filename to upload to
upload_to = "Singularity.%s-%s-%s" % (
names["collection"],
names["image"],
names["tag"],
)
# Prepare build request
url = "%s/%s/build/" % (names["registry"].replace("/api",
""), builder_type)
SREGISTRY_EVENT = self.authorize(request_type="build", names=names)
headers = {"Authorization": SREGISTRY_EVENT}
bot.debug("Setting build URL to {0}".format(url))
# Fields for build endpoint
fields = {
"SREGISTRY_EVENT": SREGISTRY_EVENT,
"name": names["image"],
"collection": names["collection"],
"tag": names["tag"],
"datafile": (upload_to, open(path, "rb"), "text/plain"),
}
encoder = MultipartEncoder(fields=fields)
progress_callback = create_callback(encoder, self.quiet)
monitor = MultipartEncoderMonitor(encoder, progress_callback)
headers = {
"Content-Type": monitor.content_type,
"Authorization": SREGISTRY_EVENT
}
try:
r = requests.post(url, data=monitor, headers=headers)
r.raise_for_status()
print("\n[Return status {0} Created]".format(r.status_code))
except requests.HTTPError as e:
print("\nRecipe upload failed: {0}.".format(e))
except KeyboardInterrupt:
print("\nRecipe upload cancelled.")
def push(self, path, name, tag=None):
'''push an image to Singularity Registry'''
path = os.path.abspath(path)
image = os.path.basename(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' %path)
sys.exit(1)
# Interaction with a registry requires secrets
self.require_secrets()
# Extract the metadata
names = parse_image_name(remove_uri(name), tag=tag)
metadata = self.get_metadata(path, names=names)
# Try to add the size
try:
image_size = os.path.getsize(path) >> 20
if metadata['data']['attributes']['labels'] is None:
metadata['data']['attributes']['labels'] = {'SREGISTRY_SIZE_MB': image_size }
else:
metadata['data']['attributes']['labels']['SREGISTRY_SIZE_MB'] = image_size
except:
bot.warning("Cannot load metadata to add calculated size.")
pass
if "deffile" in metadata['data']['attributes']:
if metadata['data']['attributes']['deffile'] is not None:
fromimage = parse_header(metadata['data']['attributes']['deffile'],
header="from",
remove_header=True)
metadata['data']['attributes']['labels']['SREGISTRY_FROM'] = fromimage
bot.debug("%s was built from a definition file." % image)
# Prepare push request with multipart encoder
url = '%s/push/' % self.base
upload_to = os.path.basename(names['storage'])
SREGISTRY_EVENT = self.authorize(request_type="push",
names=names)
encoder = MultipartEncoder(fields={'collection': names['collection'],
'name':names['image'],
'metadata': json.dumps(metadata),
'tag': names['tag'],
'datafile': (upload_to, open(path, 'rb'), 'text/plain')})
progress_callback = create_callback(encoder)
monitor = MultipartEncoderMonitor(encoder, progress_callback)
headers = {'Content-Type': monitor.content_type,
'Authorization': SREGISTRY_EVENT }
try:
r = requests.post(url, data=monitor, headers=headers)
message = self._read_response(r)
print('\n[Return status {0} {1}]'.format(r.status_code, message))
except KeyboardInterrupt:
print('\nUpload cancelled.')
def push(self, path, name, tag=None):
'''push an image to Singularity Registry'''
path = os.path.abspath(path)
image = os.path.basename(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' % path)
sys.exit(1)
# Interaction with a registry requires secrets
self.require_secrets()
# Extract the metadata
names = parse_image_name(remove_uri(name), tag=tag)
image_size = os.path.getsize(path) >> 20
# COLLECTION ###################################################################
# Prepare push request, this will return a collection ID if permission
url = '%s/push/' % self.base
auth_url = '%s/upload/chunked_upload' % self.base
SREGISTRY_EVENT = self.authorize(request_type="push", names=names)
# Data fields for collection
fields = {
'collection': names['collection'],
'name': names['image'],
'tag': names['tag']
}
headers = {'Authorization': SREGISTRY_EVENT}
r = requests.post(auth_url, json=fields, headers=headers)
# Always tell the user what's going on!
message = self._read_response(r)
print('\n[1. Collection return status {0} {1}]'.format(
r.status_code, message))
# Get the collection id, if created, and continue with upload
if r.status_code != 200:
sys.exit(1)
# UPLOAD #######################################################################
url = '%s/upload' % self.base.replace('/api', '')
bot.debug('Seting upload URL to {0}'.format(url))
cid = r.json()['cid']
upload_to = os.path.basename(names['storage'])
SREGISTRY_EVENT = self.authorize(request_type="upload", names=names)
encoder = MultipartEncoder(
fields={
'SREGISTRY_EVENT': SREGISTRY_EVENT,
'name': names['image'],
'collection': str(cid),
'tag': names['tag'],
'file1': (upload_to, open(path, 'rb'), 'text/plain')
})
progress_callback = create_callback(encoder, self.quiet)
monitor = MultipartEncoderMonitor(encoder, progress_callback)
headers = {
'Content-Type': monitor.content_type,
'Authorization': SREGISTRY_EVENT
}
try:
r = requests.post(url, data=monitor, headers=headers)
r.raise_for_status()
message = r.json()['message']
print('\n[Return status {0} {1}]'.format(r.status_code, message))
except requests.HTTPError as e:
print('\nUpload failed: {0}.'.format(e))
except KeyboardInterrupt:
print('\nUpload cancelled.')
except Exception as e:
print(e)
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from google storage, based on the identifier
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
q = parse_image_name(remove_uri(image))
# Use container search to find the container based on uri
bot.info('Searching for %s in gs://%s' %
(q['tag_uri'], self._bucket_name))
matches = self._container_query(q['tag_uri'], quiet=True)
if len(matches) == 0:
bot.info('No matching containers found.')
sys.exit(0)
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = q['storage_version'].replace('/', '-')
# We give the first match, the uri should be unique and known
image = matches[0]
image_file = self.download(url=image.media_link,
file_name=file_name,
show_progress=True)
# If the user is saving to local storage, you need to assumble the uri
# here in the expected format <collection>/<namespace>:<tag>@<version>
if save is True:
image_uri = q['tag_uri']
# Update metadata with selfLink
metadata = image.metadata
# Rename public URL to URL so it's found by add client
if "public_url" in metadata:
metadata['url'] = metadata['public_url']
metadata['selfLink'] = image.self_link
container = self.add(image_path=image_file,
image_uri=image_uri,
metadata=metadata,
url=image.media_link)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished