def container_query(self, query, quiet=False):
'''search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
'''
results = self._list_containers()
matches = []
for result in results:
for key, val in result.metadata.items():
if query in val and result not in matches:
matches.append(result)
if not quiet:
bot.info("[gs://%s] Found %s containers" %
(self._bucket_name, len(matches)))
for image in matches:
size = round(image.size / (1024 * 1024.0))
bot.custom(prefix=image.name, color="CYAN")
bot.custom(prefix='id: ', message=image.id)
bot.custom(prefix='uri: ', message=image.metadata['uri'])
bot.custom(prefix='updated:', message=image.updated)
bot.custom(prefix='size: ', message=' %s MB' % (size))
bot.custom(prefix='md5: ', message=image.md5_hash)
bot.newline()
return matches
def list_containers(self):
'''return a list of containers, determined by finding the metadata field
"type" with value "container." We alert the user to no containers
if results is empty, and exit
{'metadata': {'items':
[
{'key': 'type', 'value': 'container'}, ...
]
}
}
'''
results = []
for image in self._bucket.list_blobs():
if image.metadata is not None:
if "type" in image.metadata:
if image.metadata['type'] == "container":
results.append(image)
if len(results) == 0:
bot.info("No containers found, based on metadata type:container")
sys.exit(1)
return results
def _update_secrets(self):
"""update secrets will take a secrets credential file
either located at .sregistry or the environment variable
SREGISTRY_CLIENT_SECRETS and update the current client
secrets as well as the associated API base. This is where you
should do any customization of the secrets flie, or using
it to update your client, if needed.
"""
# Get a setting for client myclient and some variable name VAR.
# returns None if not set
# setting = self._get_setting('SREGISTRY_MYCLIENT_VAR')
# Get (and if found in environment (1) settings (2) update the variable
# It will still return None if not set
# setting = self._get_and_update_setting('SREGISTRY_MYCLIENT_VAR')
# If you have a setting that is required and not found, you should exit.
# Here is how to read all client secrets
self.secrets = read_client_secrets()
# If you don't want to use the shared settings file, you have your own.
# Here is how to get if the user has a cache for you enabled, this
# returns a path (enabled) or None (disabled) that you should honor
# You can use this as a file path or folder and for both cases, you
# need to create the file or folder
if self._credential_cache is not None:
bot.info("credential cache set to %s" % self._credential_cache)
def list_all(self, **kwargs):
"""a "show all" search that doesn't require a query"""
quiet = False
if "quiet" in kwargs:
quiet = kwargs["quiet"]
bot.spinner.start()
url = "%s/collections/" % self.base
results = self._paginate_get(url)
bot.spinner.stop()
if len(results) == 0:
bot.exit("No container collections found.", return_code=0)
rows = []
for result in results:
if "containers" in result:
if result["id"] not in [37, 38, 39]:
for c in result["containers"]:
rows.append([c["detail"], "%s:%s" % (c["name"], c["tag"])])
if quiet is False:
bot.info("Collections")
bot.table(rows)
return rows
def remove(self, image, force=False):
'''delete an image to Singularity Registry'''
q = parse_image_name(remove_uri(image))
url = '%s/container/%s/%s:%s' % (self.base, q["collection"], q["image"], q["tag"])
SREGISTRY_EVENT = self.authorize(request_type="delete", names=q)
headers = {'Authorization': SREGISTRY_EVENT }
self._update_headers(fields=headers)
continue_delete = True
if force is False:
response = input("Are you sure you want to delete %s?" % q['uri'])
while len(response) < 1 or response[0].lower().strip() not in "ynyesno":
response = input("Please answer yes or no: ")
if response[0].lower().strip() in "no":
continue_delete = False
if continue_delete is True:
response = self._delete(url)
message = self._read_response(response)
bot.info("Response %s, %s" %(response.status_code, message))
else:
bot.info("Delete cancelled.")
def share(self, query, share_to):
'''share will use the client to get an image based on a query, and then
the link with an email or endpoint (share_to) of choice.
'''
images = self._container_query(query, quiet=True)
if len(images) == 0:
bot.info('Cannot find a remote image matching %s' % query)
sys.exit(0)
image = images[0]
def callback(request_id, response, exception):
if exception:
# Handle error
print(exception)
else:
share_id = response.get('id')
bot.info('Share to %s complete: %s!' % (share_to, share_id))
batch = self._service.new_batch_http_request(callback=callback)
user_permission = {
'type': 'user',
'role': 'reader',
'emailAddress': share_to
}
batch.add(self._service.permissions().create(
fileId=image['id'],
body=user_permission,
fields='id',
))
batch.execute()
return image
def container_query(self, query):
"""search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
"""
results = set()
query = remove_uri(query)
# Here we get names of collections, and then look up containers
for container in self.conn.get_account()[1]:
# The result here is just the name
for result in self.conn.get_container(container["name"])[1]:
if query in result["name"]:
results.add("%s/%s" % (container["name"], result["name"]))
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table([list(results)])
return list(results)
def get(self, name, quiet=False):
'''Do a get for a container, and then a collection, and then return None
if no result is found.
Parameters
==========
name: should coincide with either the collection name, or the container
name with the collection. A query is done first for the collection,
and then the container, and the path to the image file returned.
'''
from sregistry.database.models import Collection, Container
names = parse_image_name(remove_uri(name))
# First look for a collection (required)
collection = self.get_collection(name=names['collection'])
container = None
if collection is not None:
container = self.get_container(collection_id=collection.id,
name=names['image'],
tag=names['tag'],
version=names['version'])
if container is not None and quiet is False:
# The container image file exists [local]
if container.image is not None:
print(container.image)
# The container has a url (but not local file)
elif container.url is not None:
print(container.url)
else:
bot.info('No storage file found for %s' % name)
return container
def _speak(self):
'''if you want to add an extra print (of a parameter, for example)
for the user when the client initalizes, write it here, eg:
bot.info('[setting] value')
'''
if hasattr(self, 'account'):
bot.info('connected to %s' %self.account.name.display_name)
def delete(self, image, force=False):
"""delete an image to Singularity Registry"""
q = parse_image_name(remove_uri(image))
# If the registry is provided in the uri, use it
if q["registry"] is None:
q["registry"] = self.base
# If the base doesn't start with http or https, add it
q = self._add_https(q)
url = "%s/container/%s/%s:%s" % (
q["registry"],
q["collection"],
q["image"],
q["tag"],
)
SREGISTRY_EVENT = self.authorize(request_type="delete", names=q)
headers = {"Authorization": SREGISTRY_EVENT}
self._update_headers(fields=headers)
if confirm_delete(q["uri"], force) is True:
response = self._delete(url)
message = self._read_response(response)
bot.info("Response %s, %s" % (response.status_code, message))
# add some error handling here??
else:
bot.info("Delete cancelled.")
return image
def share(self, query, share_to):
"""share will use the client to get an image based on a query, and then
the link with an email or endpoint (share_to) of choice.
"""
images = self._container_query(query, quiet=True)
if len(images) == 0:
bot.info("Cannot find a remote image matching %s" % query)
sys.exit(0)
image = images[0]
def callback(request_id, response, exception):
if exception:
# Handle error
print(exception)
else:
share_id = response.get("id")
bot.info("Share to %s complete: %s!" % (share_to, share_id))
batch = self._service.new_batch_http_request(callback=callback)
user_permission = {"type": "user", "role": "reader", "emailAddress": share_to}
batch.add(
self._service.permissions().create(
fileId=image["id"], body=user_permission, fields="id"
)
)
batch.execute()
return image
def container_query(self, query, quiet=False):
"""search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
"""
results = self._list_containers()
matches = []
for result in results:
for _, val in result.metadata.items():
if query in val and result not in matches:
matches.append(result)
elif query in result.name and result not in matches:
matches.append(result)
if not quiet:
bot.info("[gs://%s] Found %s containers" % (self._bucket_name, len(matches)))
for image in matches:
size = round(image.size / (1024 * 1024.0))
bot.custom(prefix=image.name, color="CYAN")
bot.custom(prefix="id: ", message=image.id)
bot.custom(prefix="name: ", message=image.name)
bot.custom(prefix="updated:", message=image.updated)
bot.custom(prefix="size: ", message=" %s MB" % (size))
bot.custom(prefix="md5: ", message=image.md5_hash)
if "public_url" in image.metadata:
public_url = image.metadata["public_url"]
bot.custom(prefix="url: ", message=public_url)
bot.newline()
return matches
def list_all(self, **kwargs):
'''a "show all" search that doesn't require a query'''
quiet = False
if "quiet" in kwargs:
quiet = kwargs['quiet']
bot.spinner.start()
url = '%s/collections/' % self.base
results = self._paginate_get(url)
bot.spinner.stop()
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
rows = []
for result in results:
if "containers" in result:
if result['id'] not in [37, 38, 39]:
for c in result['containers']:
rows.append([c['detail'], "%s:%s" % (c['name'], c['tag'])])
if quiet is False:
bot.info("Collections")
bot.table(rows)
return rows
def list_containers(self):
'''return a list of containers. Since Google Drive definitely has other
kinds of files, we look for containers in a special sregistry folder,
(meaning the parent folder is sregistry) and with properties of type
as container.
'''
# Get or create the base
folder = self._get_or_create_folder(self._base)
next_page = None
containers = []
# Parse the base for all containers, possibly over multiple pages
while True:
query = "mimeType='application/octet-stream'" # ensures container
query += " and properties has { key='type' and value='container' }"
query += " and '%s' in parents" %folder['id'] # ensures in parent folder
response = self._service.files().list(q=query,
spaces='drive',
fields='nextPageToken, files(id, name, properties)',
pageToken=next_page).execute()
containers += response.get('files', [])
# If there is a next page, keep going!
next_page = response.get('nextPageToken')
if not next_page:
break
if len(containers) == 0:
bot.info("No containers found, based on properties type:container")
sys.exit(0)
return containers
def _speak(self):
"""if you want to add an extra print (of a parameter, for example)
for the user when the client initalizes, write it here, eg:
bot.info('[setting] value')
"""
if hasattr(self, "account"):
bot.info("connected to %s" % self.name)
def container_query(self, query):
'''search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
'''
results = []
query = remove_uri(query)
# Parse through folders (collections):
for entry in self.dbx.files_list_folder('').entries:
# Parse through containers
for item in self.dbx.files_list_folder(entry.path_lower).entries:
name = item.name.replace('.simg', '')
name = "%s/%s" % (entry.name, name)
if query in name:
results.append([name])
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table(results)
return results
def search_all(self):
"""a "show all" search that doesn't require a query"""
# This should be your apis url for a search
url = "..."
# paginte get is what it sounds like, and what you want for multiple
# pages of results
results = self._paginate_get(url)
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
# Here is how to create a simple table. You of course must parse your
# custom result and form the fields in the table to be what you think
# are important!
rows = []
for result in results:
if "containers" in result:
for c in result["containers"]:
rows.append([c["uri"], c["detail"]])
bot.table(rows)
return rows
def get_singularity_version(singularity_version=None):
'''get_singularity_version will determine the singularity version for a
build first, an environmental variable is looked at, followed by
using the system version.
Parameters
==========
singularity_version: if not defined, look for in environment. If still
not find, try finding via executing --version to Singularity. Only return
None if not set in environment or installed.
'''
if singularity_version is None:
singularity_version = os.environ.get("SINGULARITY_VERSION")
if singularity_version is None:
try:
cmd = ['singularity','--version']
output = run_command(cmd)
if isinstance(output['message'],bytes):
output['message'] = output['message'].decode('utf-8')
singularity_version = output['message'].strip('\n')
bot.info("Singularity %s being used." % singularity_version)
except:
singularity_version = None
bot.warning("Singularity version not found, so it's likely not installed.")
return singularity_version
def callback(request_id, response, exception):
if exception:
# Handle error
print(exception)
else:
share_id = response.get('id')
bot.info('Share to %s complete: %s!' % (share_to, share_id))
def add(backend, variable, value, force=False):
"""add the variable to the config
"""
print("[add]")
settings = read_client_secrets()
# If the variable begins with the SREGISTRY_<CLIENT> don't add it
prefix = "SREGISTRY_%s_" % backend.upper()
if not variable.startswith(prefix):
variable = "%s%s" % (prefix, variable)
# All must be uppercase
variable = variable.upper()
bot.info("%s %s" % (variable, value))
# Does the setting already exist?
if backend in settings:
if variable in settings[backend] and force is False:
previous = settings[backend][variable]
bot.exit(
"%s is already set as %s. Use --force to override."
% (variable, previous)
)
if backend not in settings:
settings[backend] = {}
settings[backend][variable] = value
update_secrets(settings)
def cp(self, move_to, image_name=None, container=None, command="copy"):
'''_cp is the shared function between mv (move) and rename, and performs
the move, and returns the updated container
Parameters
==========
image_name: an image_uri to look up a container in the database
container: the container object to move (must have a container.image
move_to: the full path to move it to
'''
if container is None and image_name is None:
bot.error('A container or image_name must be provided to %s' % command)
sys.exit(1)
# If a container isn't provided, look for it from image_uri
if container is None:
container = self.get(image_name, quiet=True)
image = container.image or ''
if os.path.exists(image):
filedir = os.path.dirname(move_to)
# If the two are the same, doesn't make sense
if move_to == image:
bot.warning('%s is already the name.' % image)
sys.exit(1)
# Ensure directory exists
if not os.path.exists(filedir):
bot.error('%s does not exist. Ensure exists first.' % filedir)
sys.exit(1)
# Ensure writable for user
if not os.access(filedir, os.W_OK):
bot.error('%s is not writable' % filedir)
sys.exit(1)
original = os.path.basename(image)
try:
shutil.move(image, move_to)
container.image = move_to
self.session.commit()
bot.info('[%s] %s => %s' % (command, original, move_to))
return container
except:
bot.error('Cannot %s %s to %s' % (command, original, move_to))
sys.exit(1)
bot.warning('''This operation is not permitted on a remote image.
Please pull %s and then %s to the appropriate
location.''' % (container.uri, command))
def container_search(self, query, across_collections=False):
"""search for a specific container. If across collections is False,
the query is parsed as a full container name and a specific container
is returned. If across_collections is True, the container is searched
for across collections. If across collections is True, details are
not shown"""
query = query.lower().strip("/")
q = parse_image_name(remove_uri(query), defaults=False)
if q["tag"] is not None:
if across_collections is True:
url = "%s/container/search/name/%s/tag/%s" % (
self.base,
q["image"],
q["tag"],
)
else:
url = "%s/container/search/collection/%s/name/%s/tag/%s" % (
self.base,
q["collection"],
q["image"],
q["tag"],
)
elif q["tag"] is None:
if across_collections is True:
url = "%s/container/search/name/%s" % (self.base, q["image"])
else:
url = "%s/container/search/collection/%s/name/%s" % (
self.base,
q["collection"],
q["image"],
)
result = self._get(url)
if "containers" in result:
result = result["containers"]
if len(result) == 0:
bot.info("No containers found.")
sys.exit(1)
bot.info("Containers %s" % query)
rows = []
for c in result:
rows.append(["%s/%s" % (c["collection"], c["name"]), c["tag"]])
bot.table(rows)
return rows
def list_logs(self):
"""return a list of logs. We return any file that ends in .log
"""
results = []
for image in self._bucket.list_blobs():
if image.name.endswith("log"):
results.append(image)
if len(results) == 0:
bot.info("No containers found, based on extension .log")
return results
def list_endpoint(self, endpoint, query=None):
'''An endpoint is required here to list files within. Optionally, we can
take a path relative to the endpoint root.
Parameters
==========
endpoint: a single endpoint ID or an endpoint id and relative path.
If no path is provided, we use '', which defaults to scratch.
query: if defined, limit files to those that have query match
'''
if not hasattr(self, 'transfer_client'):
self._init_transfer_client()
# Separate endpoint id from the desired path
endpoint, path = self._parse_endpoint_name(endpoint)
# Get a list of files at endpoint, under specific path
try:
result = self.transfer_client.operation_ls(endpoint, path=path)
except TransferAPIError as err:
# Tell the user what went wrong!
bot.custom(prefix='ERROR', message=err, color='RED')
sys.exit(1)
rows = []
for filey in result:
# Highlight container contenders with purple
name = filey['name']
if query is None or query in name:
if name.endswith('img'):
name = bot.addColor('PURPLE', name)
rows.append([
filey['type'], filey['permissions'],
str(filey['size']), name
])
if len(rows) > 0:
rows = [["type", "[perm]", "[size]", "[name]"]] + rows
bot.custom(prefix="Endpoint Listing %s" % path,
message='',
color="CYAN")
bot.table(rows)
else:
bot.info('No content was found at the selected endpoint.')
return rows
def speak(self):
'''
a function for the client to announce him or herself, depending
on the level specified. If you want your client to have additional
announced things here, then implement the class `_speak` for your
client.
'''
if self.quiet is False:
bot.info('[client|%s] [database|%s]' %
(self.client_name, self.database))
self._speak()
def create_endpoint_folder(self, endpoint_id, folder):
'''create an endpoint folder, catching the error if it exists.
Parameters
==========
endpoint_id: the endpoint id parameters
folder: the relative path of the folder to create
'''
try:
res = self.transfer_client.operation_mkdir(endpoint_id, folder)
bot.info("%s --> %s" % (res['message'], folder))
except TransferAPIError:
bot.info('%s already exists at endpoint' % folder)
def main(args, parser, subparser):
from sregistry.main import get_client
for query in args.query:
if query in ['', '*']:
query = None
try:
cli = get_client(query, args.quiet)
cli.announce(args.command)
cli.search(query=query, args=args)
except NotImplementedError:
bot.info('Search is not available for this endpoint.')
def _get_or_create_collection(self, name):
'''get or create a collection, meaning that if the get returns
None, create and return the response to the user.
Parameters
==========
name: the name of the collection to get (and create)
'''
try:
collection = self._get_collection(name)
except:
bot.info('Creating collection %s...' % name)
collection = self.conn.put_container(name)
return collection
def record(self, images, action='add'):
'''record an image from an endpoint. This function is akin to a pull,
but without retrieving the image. We only care about the list of images
(uris) to look up, and then the action that the user wants to take
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve metadata for a container based on this url.
action: the action to take with the record. By default we add it, meaning
adding a record (metadata and file url) to the database. It is
recommended to place the URL for the image download under the
container.url field, and the metadata (the image manifest) should
have a selfLink to indicate where it came from.
'''
# Take a look at pull for an example of this logic.
if not isinstance(images, list):
images = [images]
bot.debug('Execution of RECORD[%s] for %s images' % (action, len(images)))
for image in images:
q = parse_image_name(remove_uri(image))
# Use container search to find the container based on uri
bot.info('Searching for %s in gs://%s' % (q['uri'], self._bucket_name))
matches = self._container_query(q['uri'], quiet=True)
if len(matches) == 0:
bot.info('No matching containers found.')
sys.exit(0)
# We give the first match, the uri should be unique and known
image = matches[0]
image_uri = q['uri']
if "uri" in image.metadata:
image_uri = image.metadata['uri']
# Update metadata with selfLink
metadata = image.metadata
metadata['selfLink'] = image.self_link
# Use add without image path so added as a record
container = self.add(image_uri=image_uri,
metadata=metadata,
url=image.media_link)
def cp(self, move_to, image_name=None, container=None, command="copy"):
"""_cp is the shared function between mv (move) and rename, and performs
the move, and returns the updated container
Parameters
==========
image_name: an image_uri to look up a container in the database
container: the container object to move (must have a container.image
move_to: the full path to move it to
"""
if not container and not image_name:
bot.exit("A container or image_name must be provided to %s" % command)
# If a container isn't provided, look for it from image_uri
if not container:
container = self.get(image_name, quiet=True)
image = container.image or ""
if os.path.exists(image):
filedir = os.path.dirname(move_to)
# If the two are the same, doesn't make sense
if move_to == image:
bot.exit("%s is already the name." % image)
# Ensure directory exists
if not os.path.exists(filedir):
bot.exit("%s does not exist. Ensure exists first." % filedir)
# Ensure writable for user
if not os.access(filedir, os.W_OK):
bot.exit("%s is not writable" % filedir)
original = os.path.basename(image)
try:
shutil.move(image, move_to)
container.image = move_to
self.session.commit()
bot.info("[%s] %s => %s" % (command, original, move_to))
return container
except:
bot.exit("Cannot %s %s to %s" % (command, original, move_to))
bot.warning("""Not found! Please pull %s and then %s to the appropriate
location.""" % (container.uri, command))