def run_build(self, config):
"""run a build, meaning inserting an instance. Retry if there is failure
Parameters
==========
config: the configuration dictionary generated by setup_build
"""
project = self._get_project()
zone = self._get_zone()
bot.custom(prefix="INSTANCE", message=config["name"], color="CYAN")
bot.info(config["description"])
response = (
self._compute_service.instances()
.insert(project=project, zone=zone, body=config)
.execute()
)
# Direct the user to the web portal with log
ipaddress = self._get_ipaddress(config["name"])
bot.info("Robot Logger: http://%s" % ipaddress)
bot.info("Allow a few minutes for web server install, beepboop!")
return response
def images(self, query=None):
'''List local images in the database, optionally with a query.
Paramters
=========
query: a string to search for in the container or collection name|tag|uri
'''
from sregistry.database.models import Collection, Container
rows = []
if query is not None:
like = "%" + query + "%"
containers = Container.query.filter(
or_(Container.name == query, Container.tag.like(like),
Container.uri.like(like), Container.name.like(like))).all()
else:
containers = Container.query.all()
if len(containers) > 0:
message = " [date] [client]\t[uri]"
bot.custom(prefix='Containers:', message=message, color="RED")
for c in containers:
uri = c.get_uri()
created_at = c.created_at.strftime('%B %d, %Y')
rows.append([created_at, " [%s]" % c.client, uri])
bot.table(rows)
return containers
def list_endpoint(self, endpoint, query=None):
'''An endpoint is required here to list files within. Optionally, we can
take a path relative to the endpoint root.
Parameters
==========
endpoint: a single endpoint ID or an endpoint id and relative path.
If no path is provided, we use '', which defaults to scratch.
query: if defined, limit files to those that have query match
'''
if not hasattr(self, 'transfer_client'):
self._init_transfer_client()
# Separate endpoint id from the desired path
endpoint, path = self._parse_endpoint_name(endpoint)
# Get a list of files at endpoint, under specific path
try:
result = self.transfer_client.operation_ls(endpoint, path=path)
except TransferAPIError as err:
# Tell the user what went wrong!
bot.custom(prefix='ERROR', message=err, color='RED')
sys.exit(1)
rows = []
for filey in result:
# Highlight container contenders with purple
name = filey['name']
if query is None or query in name:
if name.endswith('img'):
name = bot.addColor('PURPLE', name)
rows.append([
filey['type'], filey['permissions'],
str(filey['size']), name
])
if len(rows) > 0:
rows = [["type", "[perm]", "[size]", "[name]"]] + rows
bot.custom(prefix="Endpoint Listing %s" % path,
message='',
color="CYAN")
bot.table(rows)
else:
bot.info('No content was found at the selected endpoint.')
return rows
def run_build(self, config, bucket, names):
'''run a build, meaning creating a build. Retry if there is failure
'''
project = self._get_project()
# prefix, message, color
bot.custom('PROJECT', project, "CYAN")
bot.custom('BUILD ', config['steps'][0]['name'], "CYAN")
response = self._build_service.projects().builds().create(
body=config, projectId=project).execute()
build_id = response['metadata']['build']['id']
status = response['metadata']['build']['status']
bot.log("build %s: %s" % (build_id, status))
start = time.time()
while status not in ['COMPLETE', 'FAILURE', 'SUCCESS']:
time.sleep(15)
response = self._build_service.projects().builds().get(
id=build_id, projectId=project).execute()
build_id = response['id']
status = response['status']
bot.log("build %s: %s" % (build_id, status))
end = time.time()
bot.log('Total build time: %s seconds' % (round(end - start, 2)))
# If successful, update blob metadata and visibility
if status == 'SUCCESS':
# Does the user want to keep the container private?
env = 'SREGISTRY_GOOGLE_STORAGE_PRIVATE'
blob = bucket.blob(response['artifacts']['objects']['paths'][0])
# Make Public, if desired
if self._get_and_update_setting(env) == None:
blob.make_public()
response['public_url'] = blob.public_url
# Add the metadata directly to the object
update_blob_metadata(blob, response, config, bucket, names)
response['media_link'] = blob.media_link
response['size'] = blob.size
response['file_hash'] = blob.md5_hash
return response
def container_query(self, query, quiet=False):
'''search for a specific container.
This function is the same as the search all, but instead of showing all
results, filters them down based on user criteria (the query)
'''
results = self._list_containers()
matches = []
for result in results:
is_match = False
if query in result['id']:
is_match = True
elif query in result['name']:
is_match = True
else:
for key, val in result['properties'].items():
if query in val and is_match is False:
is_match = True
break
if is_match is True:
matches.append(result)
if not quiet:
bot.info("[drive://%s] Found %s containers" %
(self._base, len(matches)))
for image in matches:
# If the image has properties, show to the user
if 'properties' in image:
image.update(image['properties'])
bot.info(image['uri'])
for key in sorted(image, key=len):
val = image[key]
if isinstance(val, str):
bot.custom(prefix=key.ljust(10), message=val, color="CYAN")
bot.newline()
return matches
def submit_build(self, config):
"""run a build, meaning creating a build. Retry if there is failure
"""
project = self._get_project()
# prefix, message, color
bot.custom("PROJECT", project, "CYAN")
for i, step in enumerate(config["steps"]):
bot.custom("BUILD %s" % i, step["name"], "CYAN")
response = (self._build_service.projects().builds().create(
body=config, projectId=project).execute())
build_id = response["metadata"]["build"]["id"]
status = response["metadata"]["build"]["status"]
bot.log("build %s: %s" % (build_id, status))
return response
def list_endpoints(self, query=None):
'''list all endpoints, providing a list of endpoints to the user to
better filter the search. This function takes no arguments,
as the user has not provided an endpoint id or query.
'''
bot.info('Please select an endpoint id to query from')
endpoints = self._get_endpoints(query)
# Iterate through endpoints to provide user a list
bot.custom(prefix="Globus", message="Endpoints", color="CYAN")
rows = []
for kind, eps in endpoints.items():
for epid, epmeta in eps.items():
rows.append([epid, '[%s]' % kind, epmeta['name']])
bot.table(rows)
return rows
def submit_build(self, config):
'''run a build, meaning creating a build. Retry if there is failure
'''
project = self._get_project()
# prefix, message, color
bot.custom('PROJECT', project, "CYAN")
for i, step in enumerate(config['steps']):
bot.custom('BUILD %s' % i, step['name'], "CYAN")
response = self._build_service.projects().builds().create(
body=config, projectId=project).execute()
build_id = response['metadata']['build']['id']
status = response['metadata']['build']['status']
bot.log("build %s: %s" % (build_id, status))
return response
def collection_search(self, query):
'''collection search will list all containers for a specific
collection. We assume query is the name of a collection'''
query = query.lower().strip('/')
url = '%s/collection/%s' % (self.base, query)
result = self._get(url)
if len(result) == 0:
bot.info("No collections found.")
sys.exit(1)
bot.custom(prefix="COLLECTION", message=query)
rows = []
for container in result['containers']:
rows.append([container['uri'], container['detail']])
bot.table(rows)
return rows
def collection_search(self, query):
"""collection search will list all containers for a specific
collection. We assume query is the name of a collection"""
query = query.lower().strip("/")
url = "%s/collection/%s" % (self.base, query)
result = self._get(url)
if len(result) == 0:
bot.info("No collections found.")
sys.exit(1)
bot.custom(prefix="COLLECTION", message=query)
rows = []
for container in result["containers"]:
rows.append([container["uri"], container["detail"]])
bot.table(rows)
return rows
def search_all(self):
'''a "list all" search that doesn't require a query. Here we return to
the user all objects that have custom properties value type set to
container, which is set when the image is pushed.
IMPORTANT: the upload function adds this metadata. For a container to
be found by the client, it must have the properties value with type
as container. It also should have a "uri" in properties to show the
user, otherwise the user will have to query / download based on the id
'''
results = self._list_containers()
matches = []
bot.info("[drive://%s] Containers" % self._base)
rows = []
for i in results:
# Fallback to the image name without the extension
uri = i['name'].replace('.simg', '')
# However the properties should include the uri
if 'properties' in i:
if 'uri' in i['properties']:
uri = i['properties']['uri']
rows.append([i['id'], uri])
# Give the user back a uri
i['uri'] = uri
matches.append(i)
bot.custom(prefix=" [drive://%s]" % self._base,
message="\t\t[id]\t[uri]",
color="PURPLE")
bot.table(rows)
return matches
def pull(self, images, file_name=None, save=True, **kwargs):
"""pull an image from a s3 storage
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
if not isinstance(images, list):
images = [images]
bot.debug("Execution of PULL for %s images" % len(images))
finished = []
for image in images:
image = remove_uri(image)
names = parse_image_name(image)
if file_name is None:
file_name = names["storage"].replace("/", "-")
# Assume the user provided the correct uri to start
uri = names["storage"]
# First try to get the storage uri directly.
try:
self.bucket.download_file(uri, file_name)
# If we can't find the file, help the user
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "404":
# Case 1, image not found, but not error with API
bot.error("Cannot find %s!" % file_name)
# Try to help the user with suggestions
results = self._search_all()
if len(results) > 0:
bot.info("Did you mean:\n" % "\n".join(results))
sys.exit(1)
else:
# Case 2: error with request, exit.
bot.exit("Error downloading image %s" % image)
# if we get down here, we have a uri
found = None
for obj in self.bucket.objects.filter(Prefix=image):
if image in obj.key:
found = obj
# If we find the object, get metadata
metadata = {}
if found is not None:
metadata = found.get()["Metadata"]
# Metadata bug will capitalize all fields, workaround is to lowercase
# https://github.com/boto/boto3/issues/1709
metadata = dict((k.lower(), v) for k, v in metadata.items())
metadata.update(names)
# If the user is saving to local storage
if save is True and os.path.exists(file_name):
container = self.add(
image_path=file_name, image_uri=names["uri"], metadata=metadata
)
file_name = container.image
# If the image was pulled to either
if os.path.exists(file_name):
bot.custom(prefix="Success!", message=file_name)
finished.append(file_name)
if len(finished) == 1:
finished = finished[0]
return finished
def pull(self, images, file_name=None, save=True, force=False, **kwargs):
""" pull an image from an endpoint
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
if not isinstance(images, list):
images = [images]
bot.debug("Execution of PULL for %s images" % len(images))
finished = []
for image in images:
q = parse_image_name(remove_uri(image), lowercase=False)
# Verify image existence, and obtain id
url = "%s/container/%s/%s:%s" % (
self.base,
q["collection"],
q["image"],
q["tag"],
)
# Add the version, if provided
if q["version"] is not None:
url = "%s@%s" % (url, q["version"])
bot.debug("Retrieving manifest at %s" % url)
manifest = self._get(url)
manifest["selfLink"] = url
# If the manifest reveals a version, update names
if "version" in manifest:
q = parse_image_name(remove_uri(image), version=manifest["version"])
if file_name is None:
file_name = self._get_storage_name(q)
file_name = os.path.abspath(file_name)
# Determine if the user already has the image
if os.path.exists(file_name) and not force:
bot.exit("Image exists! Remove first, or use --force to overwrite")
show_bar = not bool(self.quiet)
image_file = self.download(
url=manifest["image"],
file_name=os.path.basename(file_name),
show_progress=show_bar,
)
# If the user is saving to local storage
if save is True:
image_uri = "%s:%s@%s" % (
manifest["name"],
manifest["tag"],
manifest["version"],
)
container = self.add(
image_path=image_file,
image_uri=image_uri,
image_name=file_name,
metadata=manifest,
url=manifest["image"],
)
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
# Reset file name back to None in case of multiple downloads
file_name = None
# If the user is only asking for one image
if len(finished) == 1:
finished = finished[0]
return finished
def print_output(response, output_file=None):
'''print the output to the console for the user. If the user wants the content
also printed to an output file, do that.
Parameters
==========
response: the response from the builder, with metadata added
output_file: if defined, write output also to file
'''
# If successful built, show container uri
if response['status'] == 'SUCCESS':
bucket = response['artifacts']['objects']['location']
obj = response['artifacts']['objects']['paths'][0]
bot.custom("MD5HASH", response['file_hash'], 'CYAN')
bot.custom("SIZE", response['size'], 'CYAN')
bot.custom(response['status'], bucket + obj, 'CYAN')
else:
bot.custom(response['status'], 'see logs for details', 'CYAN')
# Show the logs no matter what
bot.custom("LOGS", response['logUrl'], 'CYAN')
# Did the user make the container public?
if "public_url" in response:
bot.custom('URL', response['public_url'], 'CYAN')
# Does the user also need writing to an output file?
if output_file != None:
with open(output_file, 'w') as filey:
if response['status'] == 'SUCCESS':
filey.writelines('MD5HASH %s\n' % response['file_hash'])
filey.writelines('SIZE %s\n' % response['size'])
filey.writelines('%s %s%s\n' % (response['status'], bucket, obj))
filey.writelines('LOGS %s\n' % response['logUrl'])
if "public_url" in response:
filey.writelines('URL %s\n' % response['public_url'])
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from gitlab. The image is found based on the
uri that should correspond to a gitlab repository, and then
the branch, job name, artifact folder, and tag of the container.
The minimum that we need are the job id, collection, and job name. Eg:
job_id|collection|job_name (or)
job_id|collection
Parameters
==========
images: refers to the uri given by the user to pull in the format
specified above
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
force = False
if "force" in kwargs:
force = kwargs['force']
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
# Format job_id|collection|job_name
# 122056733,singularityhub/gitlab-ci'
# 122056733,singularityhub/gitlab-ci,build
job_id, collection, job_name = self._parse_image_name(image)
names = parse_image_name(remove_uri(collection))
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = self._get_storage_name(names)
# If the file already exists and force is False
if os.path.exists(file_name) and force is False:
bot.error(
'Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
# Put together the GitLab URI
image_name = "Singularity.%s.simg" % (names['tag'])
if names['tag'] == 'latest':
image_name = "Singularity.simg"
# Assemble artifact path
artifact_path = "%s/%s" % (self.artifacts, image_name)
bot.info('Looking for artifact %s for job name %s, %s' %
(artifact_path, job_name, job_id))
project = quote_plus(collection.strip('/'))
# This is supposed to work, but it doesn't
# url = "%s/projects/%s/jobs/%s/artifacts/file/%s" %(self.api_base,
# project, job_id,
# artifact_path)
# This does work :)
url = "%s/%s/-/jobs/%s/artifacts/raw/%s/?inline=false" % (
self.base, collection, job_id, artifact_path)
bot.info(url)
# stream the url content to the file name
image_file = self.download(url=url,
file_name=file_name,
show_progress=True)
metadata = self._get_metadata()
metadata['collection'] = collection
metadata['job_id'] = job_id
metadata['job_name'] = job_name
metadata['artifact_path'] = artifact_path
metadata['sregistry_pull'] = image
# If we save to storage, the uri is the dropbox_path
if save is True:
container = self.add(image_path=image_file,
image_uri=image,
metadata=metadata,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def container_query(self, query, quiet=False):
'''search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
'''
results = self._list_containers()
matches = []
for result in results:
for key, val in result.metadata.items():
if query in val and result not in matches:
matches.append(result)
if not quiet:
bot.info("[gs://%s] Found %s containers" %
(self._bucket_name, len(matches)))
for image in matches:
size = round(image.size / (1024 * 1024.0))
bot.custom(prefix=image.name, color="CYAN")
bot.custom(prefix='id: ', message=image.id)
bot.custom(prefix='uri: ', message=image.metadata['uri'])
bot.custom(prefix='updated:', message=image.updated)
bot.custom(prefix='size: ', message=' %s MB' % (size))
bot.custom(prefix='md5: ', message=image.md5_hash)
bot.newline()
return matches
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from a singularity registry
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
# Here we take an entire list or a single image by ensuring we have a list
# This makes the client flexible to command line or internal function use,
# for one or more images.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
q = parse_image_name(remove_uri(image))
# Verify image existence, and obtain id
url = "..." # write your custom endpoint URL here
bot.debug('Retrieving manifest at %s' % url)
# You can use the client get function to retrieve a url manifest
manifest = self._get(url)
# it's good practice to add the url as a `selfLink`
manifest['selfLink'] = url
# Make sure to parse the response (manifest) in case it's not what
# you expect!
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = q['storage'].replace('/', '-')
# You can then use the client download function to get the url
# for some image in your manifest. In this example, it's in the `image`
# field and we want to show the progress bar.
image_file = self.download(url=manifest['image'],
file_name=file_name,
show_progress=True)
# If the user is saving to local storage, you need to assumble the uri
# here in the expected format <collection>/<namespace>:<tag>@<version>
if save is True:
image_uri = "%s/%s:%s" % (manifest['collection'], manifest['name'],
manifest['tag'])
# Importantly, the client add function will take the image file, the
# uri, the download link, and any relevant metadata (dictionary)
# for the database
container = self.add(
image_path=image_file, # the file path
image_uri=image_uri, # the full uri
image_name=file_name, # a custom name?
metadata=manifest,
url=manifest['image'])
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def pull(self, images, file_name=None, save=True, **kwargs):
"""pull an image from a singularity registry
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
if not isinstance(images, list):
images = [images]
bot.debug("Execution of PULL for %s images" % len(images))
finished = []
for image in images:
q = parse_image_name(remove_uri(image))
# If a custom registry is not set, use default base
if q["registry"] is None:
q["registry"] = self.base
# If the registry is still None, no go
if q["registry"] is None:
bot.exit(
"You must define a base in secrets, image uri, or environment")
# Ensure https is added back to the registry uri
q = self._add_https(q)
# All custom registries need api appended
if not q["registry"].endswith("api"):
q["registry"] = "%s/api" % q["registry"]
# Verify image existence, and obtain id
url = "%s/container/%s/%s:%s" % (
q["registry"],
q["collection"],
q["image"],
q["tag"],
)
bot.debug("Retrieving manifest at %s" % url)
try:
manifest = self._get(url)
except SSLError:
bot.exit(
"Issue with %s, try exporting SREGISTRY_REGISTRY_NOHTTPS." %
url)
# Private container collection
if isinstance(manifest, Response):
# Requires token
if manifest.status_code in [403, 401]:
SREGISTRY_EVENT = self.authorize(request_type="pull", names=q)
headers = {"Authorization": SREGISTRY_EVENT}
self._update_headers(headers)
manifest = self._get(url)
# Still denied
if isinstance(manifest, Response):
if manifest.status_code == 403:
manifest = 403
if isinstance(manifest, int):
if manifest == 400:
bot.exit("Bad request (400). Is this a private container?")
elif manifest == 404:
bot.exit("Container not found (404)")
elif manifest == 403:
bot.exit("Unauthorized (403)")
# Successful pull
if "image" in manifest:
# Add self link to manifest
manifest["selfLink"] = url
if file_name is None:
file_name = q["storage"].replace("/", "-")
# Clear headers of previous token
self._reset_headers()
# Show progress if not quiet
image_file = self.download(url=manifest["image"],
file_name=file_name,
show_progress=not self.quiet)
# If the user is saving to local storage
if save is True:
image_uri = "%s/%s:%s" % (
manifest["collection"],
manifest["name"],
manifest["tag"],
)
container = self.add(
image_path=image_file,
image_uri=image_uri,
metadata=manifest,
url=manifest["image"],
)
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from google drive, based on a query (uri or id)
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
q = parse_image_name(remove_uri(image))
# Use container search to find the container based on uri
bot.info('Searching for %s in drive://%s' % (q['uri'], self._base))
matches = self._container_query(q['uri'], quiet=True)
if len(matches) == 0:
bot.info('No matching containers found.')
sys.exit(0)
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = q['storage'].replace('/', '-')
# We give the first match, the uri should be unique and known
image = matches[0]
request = self._service.files().get_media(fileId=image['id'])
with open(file_name, 'wb') as fh:
downloader = MediaIoBaseDownload(fh, request)
done = False
bar = None
# Download and update the user with progress bar
while done is False:
status, done = downloader.next_chunk()
response = None
# Create bar on first call
if bar is None:
total = status.total_size / (1024 * 1024.0)
bar = ProgressBar(expected_size=total, filled_char='=')
bar.show(status.resumable_progress / (1024 * 1024.0))
# If the user is saving to local storage, you need to assumble the uri
# here in the expected format <collection>/<namespace>:<tag>@<version>
if save is True:
image_uri = q['uri']
if "uri" in image:
image_uri = image['uri']
# Update metadata with selfLink
image['selfLink'] = downloader._uri
container = self.add(image_path=image_file,
image_uri=image_uri,
metadata=image,
url=downloader._uri)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from google storage, based on the identifier
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
q = parse_image_name(remove_uri(image))
# Use container search to find the container based on uri
bot.info('Searching for %s in gs://%s' %
(q['tag_uri'], self._bucket_name))
matches = self._container_query(q['tag_uri'], quiet=True)
if len(matches) == 0:
bot.info('No matching containers found.')
sys.exit(0)
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = q['storage_version'].replace('/', '-')
# We give the first match, the uri should be unique and known
image = matches[0]
image_file = self.download(url=image.media_link,
file_name=file_name,
show_progress=True)
# If the user is saving to local storage, you need to assumble the uri
# here in the expected format <collection>/<namespace>:<tag>@<version>
if save is True:
image_uri = q['tag_uri']
# Update metadata with selfLink
metadata = image.metadata
# Rename public URL to URL so it's found by add client
if "public_url" in metadata:
metadata['url'] = metadata['public_url']
metadata['selfLink'] = image.self_link
container = self.add(image_path=image_file,
image_uri=image_uri,
metadata=metadata,
url=image.media_link)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from a singularity registry
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
if not isinstance(images,list):
images = [images]
# Interaction with a registry requires secrets
self.require_secrets()
bot.debug('Execution of PULL for %s images' %len(images))
finished = []
for image in images:
q = parse_image_name(remove_uri(image))
# Verify image existence, and obtain id
url = "%s/container/%s/%s:%s" %(self.base, q['collection'], q['image'], q['tag'])
bot.debug('Retrieving manifest at %s' %url)
manifest = self._get(url)
# Private container collection
if isinstance(manifest, Response):
# Requires token
if manifest.status_code == 403:
SREGISTRY_EVENT = self.authorize(request_type="pull",
names=q)
headers = {'Authorization': SREGISTRY_EVENT }
self._update_headers(headers)
manifest = self._get(url)
# Still denied
if manifest.status_code == 403:
manifest = 403
if isinstance(manifest, int):
if manifest == 400:
bot.error('Bad request (400). Is this a private container?')
elif manifest == 404:
bot.error('Container not found (404)')
elif manifest == 403:
bot.error('Unauthorized (403)')
sys.exit(1)
# Successful pull
if "image" in manifest:
# Add self link to manifest
manifest['selfLink'] = url
if file_name is None:
file_name = q['storage'].replace('/','-')
image_file = self.download(url=manifest['image'],
file_name=file_name,
show_progress=True)
# If the user is saving to local storage
if save is True:
image_uri = "%s/%s:%s" %(manifest['collection'],
manifest['name'],
manifest['tag'])
container = self.add(image_path = image_file,
image_uri = image_uri,
metadata = manifest,
url = manifest['image'])
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' %image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
upload_to = os.path.basename(names['storage'])
encoder = MultipartEncoder(fields={'collection': names['collection'],
'name':names['image'],
'metadata':metadata,
'tag': names['tag'],
'datafile': (upload_to, open(upload_from, 'rb'), 'text/plain')})
progress_callback = create_callback(encoder)
monitor = MultipartEncoderMonitor(encoder, progress_callback)
headers = {'Content-Type': monitor.content_type,
'Authorization': SREGISTRY_EVENT }
try:
r = requests.post(url, data=monitor, headers=headers)
message = self._read_response(r)
print('\n[Return status {0} {1}]'.format(r.status_code, message))
except KeyboardInterrupt:
print('\nUpload cancelled.')
def pull(self, images, file_name=None, save=True, **kwargs):
'''pull an image from storage using Swift. The image is found based on the
storage uri
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
force = False
if "force" in kwargs:
force = kwargs['force']
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
names = parse_image_name(remove_uri(image))
# First try to get the collection
collection = self._get_collection(names['collection'])
if collection is None:
bot.error('Collection %s does not exist.' % names['collection'])
# Show the user collections he/she does have access to
collections = self.get_collections()
if collections:
bot.info('Collections available to you: \n%s' %'\n'.join(collections))
sys.exit(1)
# Determine if the container exists in storage
image_name = os.path.basename(names['storage'])
try:
obj_tuple = self.conn.get_object(names['collection'], image_name)
except ClientException:
bot.exit('%s does not exist.' % names['storage'])
# Give etag as version if version not defined
if names['version'] == None:
names['version'] = obj_tuple[0]['etag']
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = self._get_storage_name(names)
# If the file already exists and force is False
if os.path.exists(file_name) and force is False:
bot.error('Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
# Write to file
with open(file_name, 'wb') as filey:
filey.write(obj_tuple[1])
# If we save to storage, the uri is the dropbox_path
if save is True:
names.update(obj_tuple[0])
container = self.add(image_path = file_name,
image_uri = names['uri'],
metadata = names)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' %image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
else:
bot.error('%s does not exist. Try sregistry search to see images.' % path)
if len(finished) == 1:
finished = finished[0]
return finished
def print_output(response, output_file=None):
"""print the output to the console for the user. If the user wants the content
also printed to an output file, do that.
Parameters
==========
response: the response from the builder, with metadata added
output_file: if defined, write output also to file
"""
# If successful built, show container uri
if "status" in response:
if response["status"] == "SUCCESS":
bucket = response["artifacts"]["objects"]["location"]
obj = response["artifacts"]["objects"]["paths"][0]
bot.custom("MD5HASH", response["file_hash"], "CYAN")
bot.custom("SIZE", response["size"], "CYAN")
bot.custom(response["status"], bucket + obj, "CYAN")
else:
bot.custom(response["status"], "see logs for details", "CYAN")
# Show the logs no matter what
bot.custom("LOGS", response["logUrl"], "CYAN")
# Did the user make the container public?
if "public_url" in response:
bot.custom("URL", response["public_url"], "CYAN")
# Does the user also need writing to an output file?
if output_file is not None:
with open(output_file, "w") as filey:
if response["status"] == "SUCCESS":
filey.writelines("MD5HASH %s\n" % response["file_hash"])
filey.writelines("SIZE %s\n" % response["size"])
filey.writelines("%s %s%s\n" % (response["status"], bucket, obj))
filey.writelines("LOGS %s\n" % response["logUrl"])
if "public_url" in response:
filey.writelines("URL %s\n" % response["public_url"])
def _pull(self, file_name, names, save=True, force=False, **kwargs):
'''pull an image from aws. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers from S3
- converts to a squashfs image with build
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.error('Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
digest = names['version'] or names['tag']
# Build from sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers, url = self._download_layers(names['url'], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info('Exploding %s' % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result['return_code'] != 0:
bot.error(result['message'])
sys.exit(1)
sudo = kwargs.get('sudo', False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info('Downloading with native Singularity, please wait...')
image = image.replace('aws://', 'docker://')
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, 'manifest'):
manifest = self.manifest
container = self.add(image_path=image_file,
image_uri=names['uri'],
metadata=manifest,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def _pull(self,
file_name,
names,
save=True,
force=False,
uri="docker://",
**kwargs):
"""pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.exit("Image exists! Remove first, or use --force to overwrite")
digest = names["version"] or names["tag"]
# Build from sandbox, prefix with sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers = self._download_layers(names["url"], digest)
# This is the url where the manifests were obtained
url = self._get_manifest_selfLink(names["url"], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info("Exploding %s" % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result["return_code"] != 0:
bot.exit(result["message"])
sudo = kwargs.get("sudo", False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info("Downloading with native Singularity, please wait...")
image = file_name.replace("docker://", uri)
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, "manifests"):
manifests = self.manifests
container = self.add(image_path=image_file,
image_uri=names["uri"],
metadata=manifests,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def container_query(self, query, quiet=False):
"""search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
"""
results = self._list_containers()
matches = []
for result in results:
for _, val in result.metadata.items():
if query in val and result not in matches:
matches.append(result)
elif query in result.name and result not in matches:
matches.append(result)
if not quiet:
bot.info("[gs://%s] Found %s containers" % (self._bucket_name, len(matches)))
for image in matches:
size = round(image.size / (1024 * 1024.0))
bot.custom(prefix=image.name, color="CYAN")
bot.custom(prefix="id: ", message=image.id)
bot.custom(prefix="name: ", message=image.name)
bot.custom(prefix="updated:", message=image.updated)
bot.custom(prefix="size: ", message=" %s MB" % (size))
bot.custom(prefix="md5: ", message=image.md5_hash)
if "public_url" in image.metadata:
public_url = image.metadata["public_url"]
bot.custom(prefix="url: ", message=public_url)
bot.newline()
return matches
def pull(self, images, file_name=None, save=True, **kwargs):
"""pull an image from a dropbox. The image is found based on the storage
uri
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
force = False
if "force" in kwargs:
force = kwargs["force"]
if not isinstance(images, list):
images = [images]
bot.debug("Execution of PULL for %s images" % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
names = parse_image_name(remove_uri(image))
# Dropbox path is the path in storage with a slash
dropbox_path = "/%s" % names["storage"]
# If the user didn't provide a file, make one based on the names
if file_name is None:
file_name = self._get_storage_name(names)
# If the file already exists and force is False
if os.path.exists(file_name) and force is False:
bot.exit("Image exists! Remove first, or use --force to overwrite")
# First ensure that exists
if self.exists(dropbox_path) is True:
# _stream is a function to stream using the response to start
metadata, response = self.dbx.files_download(dropbox_path)
image_file = self._stream(response, stream_to=file_name)
# parse the metadata (and add inspected image)
metadata = self._get_metadata(image_file, metadata)
# If we save to storage, the uri is the dropbox_path
if save is True:
container = self.add(
image_path=image_file,
image_uri=dropbox_path.strip("/"),
metadata=metadata,
url=response.url,
)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
else:
bot.error(
"%s does not exist. Try sregistry search to see images." %
dropbox_path)
if len(finished) == 1:
finished = finished[0]
return finished
def pull(self,
images,
file_name=None,
save=True,
force=False,
base=None,
**kwargs):
'''pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
base: the registry base, in case the client doesn't want to set in env.
Returns
=======
finished: a single container path, or list of paths
'''
if not isinstance(images, list):
images = [images]
bot.debug('Execution of PULL for %s images' % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
# 0. Update the base in case we aren't working with default
base = self._update_base(image)
q = parse_image_name(remove_uri(image), base=base)
digest = q['version'] or q['tag']
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(q)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.error(
'Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
# This is the url where the manifests are obtained
url = self._get_manifest_selfLink(q['url'], digest)
# This is the Docker Hub namespace and repository
layers = self._download_layers(q['url'], digest)
# Create client to build from sandbox
cli = Singularity()
# Build from sandbox
sandbox = tempfile.mkdtemp()
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info('Exploding %s' % layer)
result = extract_tar(layer, sandbox)
if result['return_code'] != 0:
bot.error(result['message'])
sys.exit(1)
if os.geteuid() == 0:
image_file = cli.build(file_name, sandbox)
else:
image_file = cli.build(file_name, sandbox, sudo=False)
# Save to local storage
if save is True:
container = self.add(image_path=image_file,
image_uri=q['uri'],
metadata=self.manifests,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
# If the image_file is different from sandbox, remove sandbox
if image_file != sandbox:
shutil.rmtree(sandbox)
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
