def container_query(self, query):
'''search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
'''
results = []
query = remove_uri(query)
# Parse through folders (collections):
for entry in self.dbx.files_list_folder('').entries:
# Parse through containers
for item in self.dbx.files_list_folder(entry.path_lower).entries:
name = item.name.replace('.simg', '')
name = "%s/%s" % (entry.name, name)
if query in name:
results.append([name])
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table(results)
return results
def container_query(self, query):
"""search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
"""
results = set()
query = remove_uri(query)
# Here we get names of collections, and then look up containers
for container in self.conn.get_account()[1]:
# The result here is just the name
for result in self.conn.get_container(container["name"])[1]:
if query in result["name"]:
results.add("%s/%s" % (container["name"], result["name"]))
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table([list(results)])
return list(results)
def list_all(self, **kwargs):
"""a "show all" search that doesn't require a query"""
quiet = False
if "quiet" in kwargs:
quiet = kwargs["quiet"]
bot.spinner.start()
url = "%s/collections/" % self.base
results = self._paginate_get(url)
bot.spinner.stop()
if len(results) == 0:
bot.exit("No container collections found.", return_code=0)
rows = []
for result in results:
if "containers" in result:
if result["id"] not in [37, 38, 39]:
for c in result["containers"]:
rows.append([c["detail"], "%s:%s" % (c["name"], c["tag"])])
if quiet is False:
bot.info("Collections")
bot.table(rows)
return rows
def list_all(self, **kwargs):
'''a "show all" search that doesn't require a query'''
quiet = False
if "quiet" in kwargs:
quiet = kwargs['quiet']
bot.spinner.start()
url = '%s/collections/' % self.base
results = self._paginate_get(url)
bot.spinner.stop()
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
rows = []
for result in results:
if "containers" in result:
if result['id'] not in [37, 38, 39]:
for c in result['containers']:
rows.append([c['detail'], "%s:%s" % (c['name'], c['tag'])])
if quiet is False:
bot.info("Collections")
bot.table(rows)
return rows
def images(self, query=None):
'''List local images in the database, optionally with a query.
Paramters
=========
query: a string to search for in the container or collection name|tag|uri
'''
from sregistry.database.models import Collection, Container
rows = []
if query is not None:
like = "%" + query + "%"
containers = Container.query.filter(
or_(Container.name == query, Container.tag.like(like),
Container.uri.like(like), Container.name.like(like))).all()
else:
containers = Container.query.all()
if len(containers) > 0:
message = " [date] [client]\t[uri]"
bot.custom(prefix='Containers:', message=message, color="RED")
for c in containers:
uri = c.get_uri()
created_at = c.created_at.strftime('%B %d, %Y')
rows.append([created_at, " [%s]" % c.client, uri])
bot.table(rows)
return containers
def search_all(self):
"""a "show all" search that doesn't require a query"""
# This should be your apis url for a search
url = "..."
# paginte get is what it sounds like, and what you want for multiple
# pages of results
results = self._paginate_get(url)
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
# Here is how to create a simple table. You of course must parse your
# custom result and form the fields in the table to be what you think
# are important!
rows = []
for result in results:
if "containers" in result:
for c in result["containers"]:
rows.append([c["uri"], c["detail"]])
bot.table(rows)
return rows
def container_query(self, query):
"""search for a specific container.
This function would likely be similar to the above, but have different
filter criteria from the user (based on the query)
"""
# Write your functions here (likely a derivation of search_all) to do
# a more specific search. The entrypoint to both these last functions
# is via the main search function.
rows = ["", "", ""]
bot.table(rows)
return rows
def container_search(self, query, across_collections=False):
"""search for a specific container. If across collections is False,
the query is parsed as a full container name and a specific container
is returned. If across_collections is True, the container is searched
for across collections. If across collections is True, details are
not shown"""
query = query.lower().strip("/")
q = parse_image_name(remove_uri(query), defaults=False)
if q["tag"] is not None:
if across_collections is True:
url = "%s/container/search/name/%s/tag/%s" % (
self.base,
q["image"],
q["tag"],
)
else:
url = "%s/container/search/collection/%s/name/%s/tag/%s" % (
self.base,
q["collection"],
q["image"],
q["tag"],
)
elif q["tag"] is None:
if across_collections is True:
url = "%s/container/search/name/%s" % (self.base, q["image"])
else:
url = "%s/container/search/collection/%s/name/%s" % (
self.base,
q["collection"],
q["image"],
)
result = self._get(url)
if "containers" in result:
result = result["containers"]
if len(result) == 0:
bot.info("No containers found.")
sys.exit(1)
bot.info("Containers %s" % query)
rows = []
for c in result:
rows.append(["%s/%s" % (c["collection"], c["name"]), c["tag"]])
bot.table(rows)
return rows
def list_endpoint(self, endpoint, query=None):
'''An endpoint is required here to list files within. Optionally, we can
take a path relative to the endpoint root.
Parameters
==========
endpoint: a single endpoint ID or an endpoint id and relative path.
If no path is provided, we use '', which defaults to scratch.
query: if defined, limit files to those that have query match
'''
if not hasattr(self, 'transfer_client'):
self._init_transfer_client()
# Separate endpoint id from the desired path
endpoint, path = self._parse_endpoint_name(endpoint)
# Get a list of files at endpoint, under specific path
try:
result = self.transfer_client.operation_ls(endpoint, path=path)
except TransferAPIError as err:
# Tell the user what went wrong!
bot.custom(prefix='ERROR', message=err, color='RED')
sys.exit(1)
rows = []
for filey in result:
# Highlight container contenders with purple
name = filey['name']
if query is None or query in name:
if name.endswith('img'):
name = bot.addColor('PURPLE', name)
rows.append([
filey['type'], filey['permissions'],
str(filey['size']), name
])
if len(rows) > 0:
rows = [["type", "[perm]", "[size]", "[name]"]] + rows
bot.custom(prefix="Endpoint Listing %s" % path,
message='',
color="CYAN")
bot.table(rows)
else:
bot.info('No content was found at the selected endpoint.')
return rows
def search_all(self, quiet=False):
"""a "show all" search that doesn't require a query
Parameters
==========
quiet: if quiet is True, we only are using the function to return
rows of results.
"""
results = []
for obj in self.bucket.objects.all():
subsrc = obj.Object()
# Metadata bug will capitalize all fields, workaround is to lowercase
# https://github.com/boto/boto3/issues/1709
try:
metadata = dict((k.lower(), v) for k, v in subsrc.metadata.items())
except botocore.exceptions.ClientError as e:
bot.warning("Could not get metadata for {}: {}".format(
subsrc.key, str(e)))
continue
size = ""
# MM-DD-YYYY
datestr = "%s-%s-%s" % (
obj.last_modified.month,
obj.last_modified.day,
obj.last_modified.year,
)
if "sizemb" in metadata:
size = "%sMB" % metadata["sizemb"]
results.append([obj.key, datestr, size])
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
if not quiet:
bot.info("Containers")
bot.table(results)
return results
def label_search(self, key=None, value=None):
'''search across labels'''
if key is not None:
key = key.lower()
if value is not None:
value = value.lower()
show_details = True
if key is None and value is None:
url = '%s/labels/search' % (self.base)
show_details = False
elif key is not None and value is not None:
url = '%s/labels/search/%s/key/%s/value' % (self.base, key, value)
elif key is None:
url = '%s/labels/search/%s/value' % (self.base, value)
else:
url = '%s/labels/search/%s/key' % (self.base, key)
result = self._get(url)
if len(result) == 0:
bot.info("No labels found.")
sys.exit(0)
bot.info("Labels\n")
rows = []
for l in result:
if show_details is True:
entry = [
"%s:%s" % (l['key'], l['value']),
"\n%s\n\n" % "\n".join(l['containers'])
]
else:
entry = [
"N=%s" % len(l['containers']),
"%s:%s" % (l['key'], l['value'])
]
rows.append(entry)
bot.table(rows)
return rows
def search_all(self):
"""a "show all" search that doesn't require a query"""
results = set()
# Here we get names of collections, and then look up containers
for container in self.conn.get_account()[1]:
# The result here is just the name
for result in self.conn.get_container(container["name"])[1]:
results.add("%s/%s" % (container["name"], result["name"]))
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table([[x] for x in list(results)])
return list(results)
def list_builders(self, project=None, zone='us-west1-a'):
'''list builders, or instances for the project. They should start with
sregistry-builder
Parameters
==========
project: specify a project, will default to environment first
zone: the zone to use, defaults to us-west1-a if environment not set
'''
builders = []
instances = self._get_instances(project, zone)
for instance in instances['items']:
builders.append([instance['name'], instance['status']])
bot.info("[google-compute] Found %s instances" % (len(builders)))
bot.table(builders)
bot.newline()
def list_endpoints(self, query=None):
'''list all endpoints, providing a list of endpoints to the user to
better filter the search. This function takes no arguments,
as the user has not provided an endpoint id or query.
'''
bot.info('Please select an endpoint id to query from')
endpoints = self._get_endpoints(query)
# Iterate through endpoints to provide user a list
bot.custom(prefix="Globus", message="Endpoints", color="CYAN")
rows = []
for kind, eps in endpoints.items():
for epid, epmeta in eps.items():
rows.append([epid, '[%s]' % kind, epmeta['name']])
bot.table(rows)
return rows
def search_all(self, collection, job_id=None):
"""a "show all" search that doesn't require a query
the user is shown URLs to
"""
results = [["job_id", "browser"]]
url = "%s/projects/%s/jobs" % (self.api_base,
quote_plus(collection.strip("/")))
response = requests.get(url, headers=self.headers)
if response.status_code == 200:
jobs = response.json()
# We can't get a listing of artifacts
# https://gitlab.com/gitlab-org/gitlab-ce/issues/51515
# Parse through jobs (each can have different tags for a collection):
for job in jobs:
# Only show jobs that are successful
if job["status"] == "success":
name = job["name"]
for artifact in job["artifacts"]:
if artifact["filename"].endswith("zip"):
# The user must browse to see the names
artifact_url = "%s/%s/-/jobs/%s/artifacts/browse/%s" % (
self.base,
collection,
job["id"],
name,
)
results.append([str(job["id"]), artifact_url])
if len(results) == 1:
bot.info("No potential archives found in artifacts.")
sys.exit(0)
bot.info("Artifact Browsers (you will need path and job id for pull)")
bot.table(results)
return results
def search_all(self):
'''a "list all" search that doesn't require a query. Here we return to
the user all objects that have custom metadata value of "container"
IMPORTANT: the upload function adds this metadata. For a container to
be found by the client, it must have the type as container in metadata.
'''
results = self._list_containers()
bot.info("[gs://%s] Containers" % self._bucket_name)
rows = []
for i in results:
size = round(i.size / (1024 * 1024.0))
size = ("%s MB" % size).rjust(10)
rows.append([size, i.metadata['uri']])
bot.table(rows)
return rows
def search_collection(self, query):
"""collection search will list all containers for a specific
collection. We assume query is the name of a collection"""
query = query.lower().strip("/")
# Workaround for now - the Singularity Hub search endpoind needs fixing
containers = self.list(quiet=True)
rows = []
for result in containers:
if re.search(query, result[1]):
rows.append(result)
if len(rows) > 0:
bot.table(rows)
else:
bot.info("No containers found.")
return rows
def collection_search(self, query):
"""collection search will list all containers for a specific
collection. We assume query is the name of a collection"""
query = query.lower().strip("/")
url = "%s/collection/%s" % (self.base, query)
result = self._get(url)
if len(result) == 0:
bot.info("No collections found.")
sys.exit(1)
bot.custom(prefix="COLLECTION", message=query)
rows = []
for container in result["containers"]:
rows.append([container["uri"], container["detail"]])
bot.table(rows)
return rows
def search_all(self):
'''a "show all" search that doesn't require a query'''
results = []
# Parse through folders (collections):
for entry in self.dbx.files_list_folder('').entries:
# Parse through containers
for item in self.dbx.files_list_folder(entry.path_lower).entries:
name = item.name.replace('.simg', '')
results.append(["%s/%s" % (entry.name, name)])
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
bot.table(results)
return results
def collection_search(self, query):
'''collection search will list all containers for a specific
collection. We assume query is the name of a collection'''
query = query.lower().strip('/')
url = '%s/collection/%s' % (self.base, query)
result = self._get(url)
if len(result) == 0:
bot.info("No collections found.")
sys.exit(1)
bot.custom(prefix="COLLECTION", message=query)
rows = []
for container in result['containers']:
rows.append([container['uri'], container['detail']])
bot.table(rows)
return rows
def search_collection(self, query):
'''collection search will list all containers for a specific
collection. We assume query is the name of a collection'''
query = query.lower().strip('/')
q = parse_image_name(remove_uri(query), defaults=False)
# Workaround for now - the Singularity Hub search endpoind needs fixing
containers = self.list(quiet=True)
rows = []
for result in containers:
if re.search(query, result[1]):
rows.append(result)
if len(rows) > 0:
bot.table(rows)
else:
bot.info('No containers found.')
return rows
def search_all(self):
'''a "show all" search that doesn't require a query'''
url = '%s/collections/' % self.base
results = self._paginate_get(url)
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
rows = []
for result in results:
if "containers" in result:
for c in result['containers']:
rows.append([c['uri'], c['detail']])
bot.table(rows)
return rows
def container_search(self, query, across_collections=False):
"""search for a specific container. If across collections is False,
the query is parsed as a full container name and a specific container
is returned. If across_collections is True, the container is searched
for across collections. If across collections is True, details are
not shown"""
results = self._search_all(quiet=True)
matches = []
for result in results:
# This is the container name
if query in result[0]:
matches.append(result)
if len(matches) > 0:
bot.info("Containers %s" % query)
bot.table(matches)
else:
bot.info("No matches for %s found." % query)
return matches
def search_all(self):
'''a "list all" search that doesn't require a query. Here we return to
the user all objects that have custom properties value type set to
container, which is set when the image is pushed.
IMPORTANT: the upload function adds this metadata. For a container to
be found by the client, it must have the properties value with type
as container. It also should have a "uri" in properties to show the
user, otherwise the user will have to query / download based on the id
'''
results = self._list_containers()
matches = []
bot.info("[drive://%s] Containers" % self._base)
rows = []
for i in results:
# Fallback to the image name without the extension
uri = i['name'].replace('.simg', '')
# However the properties should include the uri
if 'properties' in i:
if 'uri' in i['properties']:
uri = i['properties']['uri']
rows.append([i['id'], uri])
# Give the user back a uri
i['uri'] = uri
matches.append(i)
bot.custom(prefix=" [drive://%s]" % self._base,
message="\t\t[id]\t[uri]",
color="PURPLE")
bot.table(rows)
return matches
def list_templates(self, name=None):
'''list templates in the builder bundle library. If a name is provided,
look it up
Parameters
==========
name: the name of a template to look up
'''
configs = self._get_templates()
rows = []
# DETAIL: The user wants to retrieve a particular configuration
if name:
matches = self._load_templates(name)
bot.debug('Found %s matches for %s' % (len(matches), name))
for match in matches:
print(json.dumps(match, indent=4, sort_keys=True))
# LISTING: If we don't have a specific name, just show all
else:
for config in configs['data']:
rows.append([config['name']])
bot.table(rows)
def search_all(self, quiet=False):
'''a "show all" search that doesn't require a query
Parameters
==========
quiet: if quiet is True, we only are using the function to return
rows of results.
'''
results = []
for obj in self.bucket.objects.all():
subsrc = obj.Object()
# Metadata bug will capitalize all fields, workaround is to lowercase
# https://github.com/boto/boto3/issues/1709
metadata = dict((k.lower(), v) for k, v in subsrc.metadata.items())
size = ''
# MM-DD-YYYY
datestr = "%s-%s-%s" % (obj.last_modified.month, obj.last_modified.day,
obj.last_modified.year)
if 'sizemb' in metadata:
size = '%sMB' % metadata['sizemb']
results.append([obj.key, datestr, size])
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
if not quiet:
bot.info("Containers")
bot.table(results)
return results
