def build(self, repo,
config=None,
name=None,
commit=None,
tag="latest",
recipe="Singularity",
preview=False):
'''trigger a build on Google Cloud (storage then compute) given a name
recipe, and Github URI where the recipe can be found.
Parameters
==========
name: should be the complete uri that the user has requested to push.
commit: a commit to use, not required, and can be parsed from URI
repo: should correspond to a Github URL or (if undefined) used local repo.
tag: a user specified tag, to take preference over tag in name
config: The local config file to use. If the file doesn't exist, then
we attempt looking up the config based on the name.
recipe: If defined, limit builder to build a single recipe
'''
bot.debug("BUILD %s" % repo)
# Ensure that repo exists (200 response)
if not self._healthy(repo):
sys.exit(1)
config = self._load_build_config(config)
# If name not provided, parse name based on repository
if name is None:
name = '/'.join(repo.split('/')[-2:])
# This returns a data structure with collection, container, based on uri
names = parse_image_name(remove_uri(name))
# First priority - user has provided a tag
names['tag'] = tag or names['tag']
# If we still don't have custom tag, check the recipe
if names['tag'] == "latest" and recipe != "Singularity":
tag = get_recipe_tag(recipe)
names = parse_image_name(remove_uri(name), tag=tag)
# The commit is the version (after the @)
commit = commit or names['version']
# Setup the build
config = self._setup_build(name=names['url'], recipe=recipe,
repo=repo, config=config,
tag=tag, commit=commit)
# The user only wants to preview the configuration
if preview is True:
return config
# Otherwise, run the build!
return self._run_build(config)
def push(self, path, name, tag=None):
"""push an image to Google Cloud Storage, meaning uploading it
Parameters
==========
path: should correspond to an absolte image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
"""
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.exit("%s does not exist." % path)
# This returns a data structure with collection, container, based on uri
names = parse_image_name(remove_uri(name), tag=tag)
if names["version"] is None:
version = get_file_hash(path, "sha256")
names = parse_image_name(remove_uri(name), tag=tag, version=version)
# Update metadata with names
metadata = self.get_metadata(path, names=names)
manifest = self._upload(source=path,
destination=names["storage"],
metadata=metadata)
print(manifest["mediaLink"])
def push(self, path, name, tag=None):
'''push an image to Google Cloud Storage, meaning uploading it
path: should correspond to an absolte image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
'''
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' % path)
sys.exit(1)
# This returns a data structure with collection, container, based on uri
names = parse_image_name(remove_uri(name), tag=tag)
if names['version'] is None:
version = get_image_hash(path)
names = parse_image_name(remove_uri(name), tag=tag, version=version)
# Update metadata with names
metadata = self.get_metadata(path, names=names)
metadata = metadata['data']
metadata.update(names)
manifest = self._upload(source=path,
destination=names['storage'],
metadata=metadata)
print(manifest['mediaLink'])
def delete(self, image, force=False):
"""delete an image to Singularity Registry"""
q = parse_image_name(remove_uri(image))
# If the registry is provided in the uri, use it
if q["registry"] is None:
q["registry"] = self.base
# If the base doesn't start with http or https, add it
q = self._add_https(q)
url = "%s/container/%s/%s:%s" % (
q["registry"],
q["collection"],
q["image"],
q["tag"],
)
SREGISTRY_EVENT = self.authorize(request_type="delete", names=q)
headers = {"Authorization": SREGISTRY_EVENT}
self._update_headers(fields=headers)
if confirm_delete(q["uri"], force) is True:
response = self._delete(url)
message = self._read_response(response)
bot.info("Response %s, %s" % (response.status_code, message))
# add some error handling here??
else:
bot.info("Delete cancelled.")
return image
def remove(self, image, force=False):
'''delete an image to Singularity Registry'''
q = parse_image_name(remove_uri(image))
url = '%s/container/%s/%s:%s' % (self.base, q["collection"], q["image"], q["tag"])
SREGISTRY_EVENT = self.authorize(request_type="delete", names=q)
headers = {'Authorization': SREGISTRY_EVENT }
self._update_headers(fields=headers)
continue_delete = True
if force is False:
response = input("Are you sure you want to delete %s?" % q['uri'])
while len(response) < 1 or response[0].lower().strip() not in "ynyesno":
response = input("Please answer yes or no: ")
if response[0].lower().strip() in "no":
continue_delete = False
if continue_delete is True:
response = self._delete(url)
message = self._read_response(response)
bot.info("Response %s, %s" %(response.status_code, message))
else:
bot.info("Delete cancelled.")
def get(self, request, name):
names = parse_image_name(name)
container = get_container(names)
# TODO: need to check permissions here
# TODO: what to return when can't find container?
if container is None:
return Response(status=404)
# Get other tags
tags = [c.tag for c in container.collection.containers.all()]
# Downloads
path = "images/%s/%s:%s" % (names['collection'], names['image'],
names['tag'])
downloads = APIRequestCount.objects.filter(
method="get", path__contains=path).count()
data = {
"deleted": False, # 2019-03-15T19:02:24.015Z
"createdAt": container.add_date.strftime(
'%Y-%m-%dT%H:%M:%S.%jZ'), # No idea what their format is...
"hash": container.version,
"size": container.metadata.get('size_mb'),
"entityName": container.name,
"collectionName": container.collection.name.split('/')[0],
"containerName": container.name.split('/')[0],
"tags": tags,
"containerStars": container.collection.star_set.count(),
"containerDownloads": downloads
}
return Response(data={"data": data}, status=200)
def get_request_collection(instance):
"""obtain the collection from a request
Parameters
==========
instance: should be an APIRequestLog object, with a response
and path to parse
"""
import pickle
pickle.dump(instance, open("instance.pkl", "wb"))
from sregistry.utils import parse_image_name
from shub.apps.main.models import Collection
try:
response = json.loads(instance.response)
name = response["collection"]
except:
# Case 1: library endpoint
if "/v1/images" in instance.path:
collection_name = instance.path.replace("/v1/images/", "")
# Case 2: shub endpoint
else:
collection_name = instance.path.replace("/api/container/", "")
name = parse_image_name(collection_name)["collection"]
try:
collection = Collection.objects.get(name=name)
except Collection.DoesNotExist:
collection = None
return collection
def container_query(q, across_collections=1):
'''query for a container depending on the provided query string'''
across_collections = bool(int(across_collections))
# Return complete lookup with image, collection, tag
q = parse_image_name(q, defaults=False)
if across_collections is True:
if q['tag'] is not None:
# Query across collections for image name and tag
return Container.objects.filter(
Q(name__contains=q['image'])
| Q(tag__contains=q['tag'])).distinct()
# Query across collections for image name
return Container.objects.filter(
Q(name__contains=q['image'])).distinct()
# Query a particular collection for image name and tag
if q['tag'] is not None:
return Collection.objects.filter(
Q(name__contains=q['image'])
| Q(collection__name__contains=q['collection'])
| Q(containers_tags__contains=q['tag'])).distinct()
# Query a particular collection for image name
return Collection.objects.filter(
Q(name__contains=q['image'])
| Q(collection__name__contains=q['collection'])).distinct()
def get(self, request, name):
print("GET DownloadImageView")
names = parse_image_name(name)
container = get_container(names)
# If no container, regardles of permissions, 404
if container is None:
return Response(status=404)
# Private containers we check the token
if container.collection.private:
token = get_token(request)
# The user is not authenticated
if not token:
return Response(status=404)
# Only owners and contributors can pull
collection = container.collection
if (token.user not in collection.owners.all()
and token.user not in collection.contributors.all()):
return Response(status=404)
# Retrieve the url for minio
storage = container.get_storage()
url = minioExternalClient.presigned_get_object(
MINIO_BUCKET,
storage,
expires=timedelta(minutes=MINIO_SIGNED_URL_EXPIRE_MINUTES),
)
return redirect(url)
def push(self, path, name, tag=None):
'''push an image to an S3 endpoint'''
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.exit('%s does not exist.' % path)
# Extract the metadata
names = parse_image_name(remove_uri(name), tag=tag)
image_size = os.path.getsize(path) >> 20
# Create extra metadata, this is how we identify the image later
# *important* bug in boto3 will return these capitalized
# see https://github.com/boto/boto3/issues/1709
metadata = {
'sizemb': "%s" % image_size,
'client': 'sregistry',
'type': 'container'
}
ExtraArgs = {"Metadata": metadata}
acl = self._get_and_update_setting('SREGISTRY_S3_OBJECT_ACL')
if acl is not None:
ExtraArgs['ACL'] = acl
try:
self.bucket.upload_file(path, names['storage'], ExtraArgs)
except botocore.exceptions.ClientError as e:
bot.exit(
"Could not upload {} to bucket. Ensure you have sufficient permissions to put objects in the bucket (s3:PutObject), as well as modify the object ACL if SREGISTRY_S3_OBJECT_ACL is set (s3:PutObjectAcl): {}"
.format(path, str(e)))
def push(self, path, name, tag=None):
'''push an image to Singularity Registry
path: should correspond to an absolte image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
'''
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' %path)
sys.exit(1)
# This returns a data structure with collection, container, based on uri
names = parse_image_name(remove_uri(name),tag=tag)
# use Singularity client, if exists, to inspect to extract metadata
metadata = self.get_metadata(path, names=names)
# If you want a spinner
bot.spinner.start()
# do your push request here. Generally you want to except a KeyboardInterrupt
# and give the user a status from the response
bot.spinner.stop()
def get(self, name, quiet=False):
'''Do a get for a container, and then a collection, and then return None
if no result is found.
Parameters
==========
name: should coincide with either the collection name, or the container
name with the collection. A query is done first for the collection,
and then the container, and the path to the image file returned.
'''
from sregistry.database.models import Collection, Container
names = parse_image_name(remove_uri(name))
# First look for a collection (required)
collection = self.get_collection(name=names['collection'])
container = None
if collection is not None:
container = self.get_container(collection_id=collection.id,
name=names['image'],
tag=names['tag'],
version=names['version'])
if container is not None and quiet is False:
# The container image file exists [local]
if container.image is not None:
print(container.image)
# The container has a url (but not local file)
elif container.url is not None:
print(container.url)
else:
bot.info('No storage file found for %s' % name)
return container
def get_metadata(self, image_file, names=None):
'''extract metadata using Singularity inspect, if the executable is found.
If not, return a reasonable default (the parsed image name)
Parameters
==========
image_file: the full path to a Singularity image
names: optional, an extracted or otherwise created dictionary of
variables for the image, likely from utils.parse_image_name
'''
if names is None:
names = {}
metadata = {}
# We can't return anything without image_file or names
if image_file:
if not os.path.exists(image_file):
bot.error('Cannot find %s.' % image_file)
return names or metadata
# The user provided a file, but no names
if not names:
names = parse_image_name(remove_uri(image_file))
# Look for the Singularity Executable
singularity = which('singularity')['message']
# Inspect the image, or return names only
if os.path.exists(singularity) and image_file:
from spython.main import Client as Singularity
# Store the original quiet setting
is_quiet = Singularity.quiet
# We try and inspect, but not required (wont work within Docker)
try:
Singularity.quiet = True
updates = Singularity.inspect(image=image_file)
except:
bot.warning(
'Inspect command not supported, metadata not included.')
updates = None
# Restore the original quiet setting
Singularity.quiet = is_quiet
# Try loading the metadata
if updates is not None:
try:
updates = json.loads(updates)
metadata.update(updates)
except:
pass
metadata.update(names)
return metadata
def pull(self,
images,
file_name=None,
save=True,
force=False,
base=None,
**kwargs):
"""pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
base: the registry base, in case the client doesn't want to set in env.
Returns
=======
finished: a single container path, or list of paths
"""
if not isinstance(images, list):
images = [images]
bot.debug("Execution of PULL for %s images" % len(images))
# If used internally we want to return a list to the user.
finished = []
for image in images:
# 0. Update the base in case we aren't working with default
base = self._update_base(image)
q = parse_image_name(remove_uri(image), base=base)
image_file = self._pull(file_name=file_name,
save=save,
force=force,
names=q,
kwargs=kwargs)
finished.append(image_file)
if len(finished) == 1:
finished = finished[0]
return finished
def get(self, request, name):
names = parse_image_name(name)
container = get_container(names)
# TODO: need to check permissions here
# TODO: what to return when can't find container?
if container is None:
return Response(status=404)
return redirect(self.get_download_url(container))
def container_search(self, query, across_collections=False):
"""search for a specific container. If across collections is False,
the query is parsed as a full container name and a specific container
is returned. If across_collections is True, the container is searched
for across collections. If across collections is True, details are
not shown"""
query = query.lower().strip("/")
q = parse_image_name(remove_uri(query), defaults=False)
if q["tag"] is not None:
if across_collections is True:
url = "%s/container/search/name/%s/tag/%s" % (
self.base,
q["image"],
q["tag"],
)
else:
url = "%s/container/search/collection/%s/name/%s/tag/%s" % (
self.base,
q["collection"],
q["image"],
q["tag"],
)
elif q["tag"] is None:
if across_collections is True:
url = "%s/container/search/name/%s" % (self.base, q["image"])
else:
url = "%s/container/search/collection/%s/name/%s" % (
self.base,
q["collection"],
q["image"],
)
result = self._get(url)
if "containers" in result:
result = result["containers"]
if len(result) == 0:
bot.info("No containers found.")
sys.exit(1)
bot.info("Containers %s" % query)
rows = []
for c in result:
rows.append(["%s/%s" % (c["collection"], c["name"]), c["tag"]])
bot.table(rows)
return rows
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
for image in images:
q = parse_image_name(remove_uri(image))
# Verify image existence, and obtain id
url = "..." # This should be some url for your endpoint to get metadata
bot.debug('Retrieving manifest at %s' % url)
# Get the manifest, add a selfLink to it (good practice)
manifest = self._get(url)
manifest['selfLink'] = url
# versions are very important! Since we aren't downloading the file,
# If you don't have a version in your manifest, don't add it to the uri.
# you will likely need to customize this string formation to make the
# expected uri as in <collection>/<namespace>:<tag>@<version>
if manifest['version'] is not None:
image_uri = "%s/%s:%s@%s" % (manifest['collection'],
manifest['name'], manifest['tag'],
manifest['version'])
else:
image_uri = "%s/%s:%s" % (manifest['collection'], manifest['name'],
manifest['tag'])
# We again use the "add" function, but we don't give an image path
# so it's just added as a record
container = self.add(image_name=image_uri,
metadata=manifest,
url=manifest['image'])
def test_parse_image_name():
print("Testing utils.parse_image_name")
from sregistry.utils import parse_image_name
names = parse_image_name('ubuntu')
ubuntu_check(names, 'ubuntu')
names = parse_image_name('ubuntu:latest')
ubuntu_check(names, 'ubuntu:latest')
names = parse_image_name('library/ubuntu:latest')
ubuntu_check(names, 'library/ubuntu:latest')
# Version should include same with version
names = parse_image_name('library/ubuntu:latest@version')
assert names['version'] == 'version'
assert names['uri'] == "library/ubuntu:latest@version"
assert names['image'] == 'ubuntu'
assert names['collection'] == 'library'
assert names['registry'] is None
assert names['tag'] == 'latest'
assert names['url'] == "library/ubuntu"
assert names['uri'] == 'library/ubuntu:latest@version'
assert names['original'] == 'library/ubuntu:latest@version'
assert names['storage'] == 'library/ubuntu:[email protected]'
def push(self, path, name, tag=None):
'''push an image to your Storage. If the collection doesn't exist,
it is created.
Parameters
==========
path: should correspond to an absolute image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
'''
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' % path)
sys.exit(1)
# Parse image names
names = parse_image_name(remove_uri(name), tag=tag)
# Get the size of the file
file_size = os.path.getsize(path)
chunk_size = 4 * 1024 * 1024
storage_path = "/%s" % names['storage']
# Create / get the collection
collection = self._get_or_create_collection(names['collection'])
# The image name is the name followed by tag
image_name = os.path.basename(names['storage'])
# prepare the progress bar
progress = 0
bot.show_progress(progress, file_size, length=35)
# Put the (actual) container into the collection
with open(path, 'rb') as F:
self.conn.put_object(names['collection'],
image_name,
contents=F.read(),
content_type='application/octet-stream')
# Finish up
bot.show_progress(iteration=file_size,
total=file_size,
length=35,
carriage_return=True)
# Newline to finish download
sys.stdout.write('\n')
def get(self, request, name):
names = parse_image_name(name)
container = get_container(names)
# TODO: need to check permissions here
# TODO: what to return when can't find container?
if container is None:
return Response(status=404)
# Private containers are simply not accessible - no way to authenticate
if container.collection.private:
return Response(status=404)
return redirect(self.get_download_url(container))
def test_parse_image_name():
print("Testing utils.parse_image_name")
from sregistry.utils import parse_image_name
names = parse_image_name("ubuntu")
ubuntu_check(names, "ubuntu")
names = parse_image_name("ubuntu:latest")
ubuntu_check(names, "ubuntu:latest")
names = parse_image_name("library/ubuntu:latest")
ubuntu_check(names, "library/ubuntu:latest")
# Version should include same with version
names = parse_image_name("library/ubuntu:latest@version")
assert names["version"] == "version"
assert names["uri"] == "library/ubuntu:latest@version"
assert names["image"] == "ubuntu"
assert names["collection"] == "library"
assert names["registry"] is None
assert names["tag"] == "latest"
assert names["url"] == "library/ubuntu"
assert names["uri"] == "library/ubuntu:latest@version"
assert names["original"] == "library/ubuntu:latest@version"
assert names["storage"] == "library/ubuntu:[email protected]"
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
for image in images:
q = parse_image_name(remove_uri(image))
# Use container search to find the container based on uri
bot.info('Searching for %s in gs://%s' % (q['uri'], self._bucket_name))
matches = self._container_query(q['uri'], quiet=True)
if len(matches) == 0:
bot.info('No matching containers found.')
sys.exit(0)
# We give the first match, the uri should be unique and known
image = matches[0]
image_uri = q['uri']
if "uri" in image.metadata:
image_uri = image.metadata['uri']
# Update metadata with selfLink
metadata = image.metadata
metadata['selfLink'] = image.self_link
# Use add without image path so added as a record
container = self.add(image_uri=image_uri,
metadata=metadata,
url=image.media_link)
def get(self, request, name):
# The request specifies ?arch=amd64 but that's all we got
print("GET GetImageView")
names = parse_image_name(name)
# If an arch is not specified, redirect to push view
arch = request.query_params.get("arch", "amd64")
container = get_container(names)
# If an arch is defined, ensure it matches the request
arch = "amd64"
if arch and container is not None:
if container.metadata.get("arch", "amd64") != "amd64":
return Response(status=404)
# If no container, regardles of permissions, 404
if container is None:
return Response(status=404)
# Private containers we check the token
if container.collection.private:
token = get_token(request)
# Only owners and contributors can pull
collection = container.collection
if (token.user not in collection.owners.all()
and token.user not in collection.contributors.all()):
return Response(status=404)
# Generate log for downloads (async with worker)
django_rq.enqueue(
generate_log,
view_name="shub.apps.api.urls.containers.ContainerDetailByName",
ipaddr=request.META.get("HTTP_X_FORWARDED_FOR", None),
method=request.method,
params=request.query_params.dict(),
request_path=request.path,
remote_addr=request.META.get("REMOTE_ADDR", ""),
host=request.get_host(),
request_data=request.data,
auth_header=request.META.get("HTTP_AUTHORIZATION"),
)
data = generate_container_metadata(container)
return Response(data={"data": data}, status=200)
def check_tasks(client, endpoint_id):
''' check on task list and compare against local, update with any containers
that have finished status
'''
from sregistry.utils import parse_image_name
events = []
for task in client.transfer_client.task_list():
tid = task.data['task_id']
# Tasks still processing won't have events available
if task.data['status'] in ["SUCCEEDED", "FAILED"]:
for event in client.transfer_client.task_successful_transfers(tid):
# Link to task history
link = "https://globus.org/app/activity/%s" %tid
dest = event['destination_path']
source = event['source_path']
# Case 1: Add images transferred here to sregistry
if task.data['destination_endpoint_id'] == endpoint_id:
# Do we have the container?
if os.path.exists(dest):
# Parse name from source_path
metadata = {'globus_event': event,
'globus_task_id': tid,
'selfLink': link }
names = parse_image_name(os.path.basename(source))
metadata.update(names)
result = client.add(image_uri=names['uri'],
image_path=dest,
metadata=metadata)
events.append(names['uri'])
# Case 2: remove temporary files for finished remote transfers
else:
if os.path.exists(event['source_path']):
os.remove(event['source_path'])
return events
def get_metadata(self, image_file, names={}):
'''extract metadata using Singularity inspect, if the executable is found.
If not, return a reasonable default (the parsed image name)
Parameters
==========
image_file: the full path to a Singularity image
names: optional, an extracted or otherwise created dictionary of
variables for the image, likely from utils.parse_image_name
'''
metadata = dict()
# We can't return anything without image_file or names
if image_file is not None:
if not os.path.exists(image_file):
bot.error('Cannot find %s.' % image_file)
return names
# The user provided a file, but no names
if not names:
names = parse_image_name(remove_uri(image_file))
# Look for the Singularity Executable
singularity = which('singularity')['message']
# Inspect the image, or return names only
if os.path.exists(singularity) and image_file is not None:
from sregistry.client import Singularity
cli = Singularity()
updates = cli.inspect(image_path=image_file, quiet=True)
# Try loading the metadata
if updates is not None:
try:
updates = json.loads(updates)
metadata.update(updates)
except:
pass
metadata.update(names)
return metadata
def rename(self, image_name, path):
'''rename performs a move, but ensures the path is maintained in storage
Parameters
==========
image_name: the image name (uri) to rename to.
path: the name to rename (basename is taken)
'''
container = self.get(image_name, quiet=True)
if container is not None:
if container.image is not None:
# The original directory for the container stays the same
dirname = os.path.dirname(container.image)
# But we derive a new filename and uri
names = parse_image_name(remove_uri(path))
storage = os.path.join(self.storage,
os.path.dirname(names['storage']))
# This is the collection folder
if not os.path.exists(storage):
os.mkdir(storage)
# Here we get the new full path, rename the container file
fullpath = os.path.abspath(os.path.join(dirname, names['storage']))
container = self.cp(move_to=fullpath,
container=container,
command="rename")
# On successful rename of file, update the uri
if container is not None:
container.uri = names['uri']
self.session.commit()
return container
bot.warning('%s not found' % (image_name))
def get(self, request, name):
print("GET DownloadImageView")
names = parse_image_name(name)
container = get_container(names)
# If no container, regardles of permissions, 404
if container is None:
return Response(status=404)
# Private containers we check the token
if container.collection.private:
token = get_token(request)
# Only owners and contributors can pull
collection = container.collection
if (token.user not in collection.owners.all()
and token.user not in collection.contributors.all()):
return Response(status=404)
return redirect(self.get_download_url(container))
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
for image in images:
names = parse_image_name(remove_uri(image))
# Dropbox path is the path in storage with a slash
dropbox_path = '/%s' % names['storage']
# First ensure that exists
if self.exists(dropbox_path) is True:
# Get metadata from dropbox, and then update with sregistry
metadata = self.dbx.files_get_metadata(dropbox_path)
metadata = self._get_metadata(dbx_metadata=metadata)
# Add image as a record
container = self.add(image_uri=dropbox_path.strip('/'),
metadata=metadata,
url=metadata['path_lower'])
def rename(self, image_name, path):
'''rename performs a move, but ensures the path is maintained in storage
Parameters
==========
image_name: the image name (uri) to rename to.
path: the name to rename (basename is taken)
'''
container = self.get(image_name, quiet=True)
if container:
if container.image:
# Derive a new filename and url in storage
names = parse_image_name(remove_uri(path),
version=container.version)
storage = self._get_storage_name(names)
dirname = os.path.dirname(storage)
# This is the collection folder
if not os.path.exists(dirname):
os.makedirs(dirname)
container = self.cp(move_to=storage,
container=container,
command="rename")
# On successful rename of file, update the uri
if container is not None:
# Create the collection if doesn't exist
collection = self.get_or_create_collection(names['collection'])
self.session.commit()
# Then update the container
container = update_container_metadata(container, collection,
names)
self.session.commit()
return container
bot.warning('%s not found' % image_name)
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
# If used internally we want to return a list to the user.
for image in images:
# 0. Update the base in case we aren't working with default
base = self._update_base(image)
q = parse_image_name(remove_uri(image), base=base)
digest = q['version'] or q['tag']
# This is the Docker Hub namespace and repository
manifests = self._get_manifests(q['url'], digest)
# This is the url where the manifests were obtained
url = self._get_manifest_selfLink(q['url'], digest)
# We again use the "add" function, but we don't give an image path
# so it's just added as a record
container = self.add(image_uri=q['uri'], metadata=manifests, url=url)
