def container_query(self, query): '''search for a specific container. This function would likely be similar to the above, but have different filter criteria from the user (based on the query) ''' results = [] query = remove_uri(query) # Parse through folders (collections): for entry in self.dbx.files_list_folder('').entries: # Parse through containers for item in self.dbx.files_list_folder(entry.path_lower).entries: name = item.name.replace('.simg', '') name = "%s/%s" % (entry.name, name) if query in name: results.append([name]) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) bot.info("Collections") bot.table(results) return results
def container_query(self, query): """search for a specific container. This function would likely be similar to the above, but have different filter criteria from the user (based on the query) """ results = set() query = remove_uri(query) # Here we get names of collections, and then look up containers for container in self.conn.get_account()[1]: # The result here is just the name for result in self.conn.get_container(container["name"])[1]: if query in result["name"]: results.add("%s/%s" % (container["name"], result["name"])) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) bot.info("Collections") bot.table([list(results)]) return list(results)
def list_all(self, **kwargs): """a "show all" search that doesn't require a query""" quiet = False if "quiet" in kwargs: quiet = kwargs["quiet"] bot.spinner.start() url = "%s/collections/" % self.base results = self._paginate_get(url) bot.spinner.stop() if len(results) == 0: bot.exit("No container collections found.", return_code=0) rows = [] for result in results: if "containers" in result: if result["id"] not in [37, 38, 39]: for c in result["containers"]: rows.append([c["detail"], "%s:%s" % (c["name"], c["tag"])]) if quiet is False: bot.info("Collections") bot.table(rows) return rows
def list_all(self, **kwargs): '''a "show all" search that doesn't require a query''' quiet = False if "quiet" in kwargs: quiet = kwargs['quiet'] bot.spinner.start() url = '%s/collections/' % self.base results = self._paginate_get(url) bot.spinner.stop() if len(results) == 0: bot.info("No container collections found.") sys.exit(1) rows = [] for result in results: if "containers" in result: if result['id'] not in [37, 38, 39]: for c in result['containers']: rows.append([c['detail'], "%s:%s" % (c['name'], c['tag'])]) if quiet is False: bot.info("Collections") bot.table(rows) return rows
def images(self, query=None): '''List local images in the database, optionally with a query. Paramters ========= query: a string to search for in the container or collection name|tag|uri ''' from sregistry.database.models import Collection, Container rows = [] if query is not None: like = "%" + query + "%" containers = Container.query.filter( or_(Container.name == query, Container.tag.like(like), Container.uri.like(like), Container.name.like(like))).all() else: containers = Container.query.all() if len(containers) > 0: message = " [date] [client]\t[uri]" bot.custom(prefix='Containers:', message=message, color="RED") for c in containers: uri = c.get_uri() created_at = c.created_at.strftime('%B %d, %Y') rows.append([created_at, " [%s]" % c.client, uri]) bot.table(rows) return containers
def search_all(self): """a "show all" search that doesn't require a query""" # This should be your apis url for a search url = "..." # paginte get is what it sounds like, and what you want for multiple # pages of results results = self._paginate_get(url) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) bot.info("Collections") # Here is how to create a simple table. You of course must parse your # custom result and form the fields in the table to be what you think # are important! rows = [] for result in results: if "containers" in result: for c in result["containers"]: rows.append([c["uri"], c["detail"]]) bot.table(rows) return rows
def container_query(self, query): """search for a specific container. This function would likely be similar to the above, but have different filter criteria from the user (based on the query) """ # Write your functions here (likely a derivation of search_all) to do # a more specific search. The entrypoint to both these last functions # is via the main search function. rows = ["", "", ""] bot.table(rows) return rows
def container_search(self, query, across_collections=False): """search for a specific container. If across collections is False, the query is parsed as a full container name and a specific container is returned. If across_collections is True, the container is searched for across collections. If across collections is True, details are not shown""" query = query.lower().strip("/") q = parse_image_name(remove_uri(query), defaults=False) if q["tag"] is not None: if across_collections is True: url = "%s/container/search/name/%s/tag/%s" % ( self.base, q["image"], q["tag"], ) else: url = "%s/container/search/collection/%s/name/%s/tag/%s" % ( self.base, q["collection"], q["image"], q["tag"], ) elif q["tag"] is None: if across_collections is True: url = "%s/container/search/name/%s" % (self.base, q["image"]) else: url = "%s/container/search/collection/%s/name/%s" % ( self.base, q["collection"], q["image"], ) result = self._get(url) if "containers" in result: result = result["containers"] if len(result) == 0: bot.info("No containers found.") sys.exit(1) bot.info("Containers %s" % query) rows = [] for c in result: rows.append(["%s/%s" % (c["collection"], c["name"]), c["tag"]]) bot.table(rows) return rows
def list_endpoint(self, endpoint, query=None): '''An endpoint is required here to list files within. Optionally, we can take a path relative to the endpoint root. Parameters ========== endpoint: a single endpoint ID or an endpoint id and relative path. If no path is provided, we use '', which defaults to scratch. query: if defined, limit files to those that have query match ''' if not hasattr(self, 'transfer_client'): self._init_transfer_client() # Separate endpoint id from the desired path endpoint, path = self._parse_endpoint_name(endpoint) # Get a list of files at endpoint, under specific path try: result = self.transfer_client.operation_ls(endpoint, path=path) except TransferAPIError as err: # Tell the user what went wrong! bot.custom(prefix='ERROR', message=err, color='RED') sys.exit(1) rows = [] for filey in result: # Highlight container contenders with purple name = filey['name'] if query is None or query in name: if name.endswith('img'): name = bot.addColor('PURPLE', name) rows.append([ filey['type'], filey['permissions'], str(filey['size']), name ]) if len(rows) > 0: rows = [["type", "[perm]", "[size]", "[name]"]] + rows bot.custom(prefix="Endpoint Listing %s" % path, message='', color="CYAN") bot.table(rows) else: bot.info('No content was found at the selected endpoint.') return rows
def search_all(self, quiet=False): """a "show all" search that doesn't require a query Parameters ========== quiet: if quiet is True, we only are using the function to return rows of results. """ results = [] for obj in self.bucket.objects.all(): subsrc = obj.Object() # Metadata bug will capitalize all fields, workaround is to lowercase # https://github.com/boto/boto3/issues/1709 try: metadata = dict((k.lower(), v) for k, v in subsrc.metadata.items()) except botocore.exceptions.ClientError as e: bot.warning("Could not get metadata for {}: {}".format( subsrc.key, str(e))) continue size = "" # MM-DD-YYYY datestr = "%s-%s-%s" % ( obj.last_modified.month, obj.last_modified.day, obj.last_modified.year, ) if "sizemb" in metadata: size = "%sMB" % metadata["sizemb"] results.append([obj.key, datestr, size]) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) if not quiet: bot.info("Containers") bot.table(results) return results
def label_search(self, key=None, value=None): '''search across labels''' if key is not None: key = key.lower() if value is not None: value = value.lower() show_details = True if key is None and value is None: url = '%s/labels/search' % (self.base) show_details = False elif key is not None and value is not None: url = '%s/labels/search/%s/key/%s/value' % (self.base, key, value) elif key is None: url = '%s/labels/search/%s/value' % (self.base, value) else: url = '%s/labels/search/%s/key' % (self.base, key) result = self._get(url) if len(result) == 0: bot.info("No labels found.") sys.exit(0) bot.info("Labels\n") rows = [] for l in result: if show_details is True: entry = [ "%s:%s" % (l['key'], l['value']), "\n%s\n\n" % "\n".join(l['containers']) ] else: entry = [ "N=%s" % len(l['containers']), "%s:%s" % (l['key'], l['value']) ] rows.append(entry) bot.table(rows) return rows
def search_all(self): """a "show all" search that doesn't require a query""" results = set() # Here we get names of collections, and then look up containers for container in self.conn.get_account()[1]: # The result here is just the name for result in self.conn.get_container(container["name"])[1]: results.add("%s/%s" % (container["name"], result["name"])) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) bot.info("Collections") bot.table([[x] for x in list(results)]) return list(results)
def list_builders(self, project=None, zone='us-west1-a'): '''list builders, or instances for the project. They should start with sregistry-builder Parameters ========== project: specify a project, will default to environment first zone: the zone to use, defaults to us-west1-a if environment not set ''' builders = [] instances = self._get_instances(project, zone) for instance in instances['items']: builders.append([instance['name'], instance['status']]) bot.info("[google-compute] Found %s instances" % (len(builders))) bot.table(builders) bot.newline()
def list_endpoints(self, query=None): '''list all endpoints, providing a list of endpoints to the user to better filter the search. This function takes no arguments, as the user has not provided an endpoint id or query. ''' bot.info('Please select an endpoint id to query from') endpoints = self._get_endpoints(query) # Iterate through endpoints to provide user a list bot.custom(prefix="Globus", message="Endpoints", color="CYAN") rows = [] for kind, eps in endpoints.items(): for epid, epmeta in eps.items(): rows.append([epid, '[%s]' % kind, epmeta['name']]) bot.table(rows) return rows
def search_all(self, collection, job_id=None): """a "show all" search that doesn't require a query the user is shown URLs to """ results = [["job_id", "browser"]] url = "%s/projects/%s/jobs" % (self.api_base, quote_plus(collection.strip("/"))) response = requests.get(url, headers=self.headers) if response.status_code == 200: jobs = response.json() # We can't get a listing of artifacts # https://gitlab.com/gitlab-org/gitlab-ce/issues/51515 # Parse through jobs (each can have different tags for a collection): for job in jobs: # Only show jobs that are successful if job["status"] == "success": name = job["name"] for artifact in job["artifacts"]: if artifact["filename"].endswith("zip"): # The user must browse to see the names artifact_url = "%s/%s/-/jobs/%s/artifacts/browse/%s" % ( self.base, collection, job["id"], name, ) results.append([str(job["id"]), artifact_url]) if len(results) == 1: bot.info("No potential archives found in artifacts.") sys.exit(0) bot.info("Artifact Browsers (you will need path and job id for pull)") bot.table(results) return results
def search_all(self): '''a "list all" search that doesn't require a query. Here we return to the user all objects that have custom metadata value of "container" IMPORTANT: the upload function adds this metadata. For a container to be found by the client, it must have the type as container in metadata. ''' results = self._list_containers() bot.info("[gs://%s] Containers" % self._bucket_name) rows = [] for i in results: size = round(i.size / (1024 * 1024.0)) size = ("%s MB" % size).rjust(10) rows.append([size, i.metadata['uri']]) bot.table(rows) return rows
def search_collection(self, query): """collection search will list all containers for a specific collection. We assume query is the name of a collection""" query = query.lower().strip("/") # Workaround for now - the Singularity Hub search endpoind needs fixing containers = self.list(quiet=True) rows = [] for result in containers: if re.search(query, result[1]): rows.append(result) if len(rows) > 0: bot.table(rows) else: bot.info("No containers found.") return rows
def collection_search(self, query): """collection search will list all containers for a specific collection. We assume query is the name of a collection""" query = query.lower().strip("/") url = "%s/collection/%s" % (self.base, query) result = self._get(url) if len(result) == 0: bot.info("No collections found.") sys.exit(1) bot.custom(prefix="COLLECTION", message=query) rows = [] for container in result["containers"]: rows.append([container["uri"], container["detail"]]) bot.table(rows) return rows
def search_all(self): '''a "show all" search that doesn't require a query''' results = [] # Parse through folders (collections): for entry in self.dbx.files_list_folder('').entries: # Parse through containers for item in self.dbx.files_list_folder(entry.path_lower).entries: name = item.name.replace('.simg', '') results.append(["%s/%s" % (entry.name, name)]) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) bot.info("Collections") bot.table(results) return results
def collection_search(self, query): '''collection search will list all containers for a specific collection. We assume query is the name of a collection''' query = query.lower().strip('/') url = '%s/collection/%s' % (self.base, query) result = self._get(url) if len(result) == 0: bot.info("No collections found.") sys.exit(1) bot.custom(prefix="COLLECTION", message=query) rows = [] for container in result['containers']: rows.append([container['uri'], container['detail']]) bot.table(rows) return rows
def search_collection(self, query): '''collection search will list all containers for a specific collection. We assume query is the name of a collection''' query = query.lower().strip('/') q = parse_image_name(remove_uri(query), defaults=False) # Workaround for now - the Singularity Hub search endpoind needs fixing containers = self.list(quiet=True) rows = [] for result in containers: if re.search(query, result[1]): rows.append(result) if len(rows) > 0: bot.table(rows) else: bot.info('No containers found.') return rows
def search_all(self): '''a "show all" search that doesn't require a query''' url = '%s/collections/' % self.base results = self._paginate_get(url) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) bot.info("Collections") rows = [] for result in results: if "containers" in result: for c in result['containers']: rows.append([c['uri'], c['detail']]) bot.table(rows) return rows
def container_search(self, query, across_collections=False): """search for a specific container. If across collections is False, the query is parsed as a full container name and a specific container is returned. If across_collections is True, the container is searched for across collections. If across collections is True, details are not shown""" results = self._search_all(quiet=True) matches = [] for result in results: # This is the container name if query in result[0]: matches.append(result) if len(matches) > 0: bot.info("Containers %s" % query) bot.table(matches) else: bot.info("No matches for %s found." % query) return matches
def search_all(self): '''a "list all" search that doesn't require a query. Here we return to the user all objects that have custom properties value type set to container, which is set when the image is pushed. IMPORTANT: the upload function adds this metadata. For a container to be found by the client, it must have the properties value with type as container. It also should have a "uri" in properties to show the user, otherwise the user will have to query / download based on the id ''' results = self._list_containers() matches = [] bot.info("[drive://%s] Containers" % self._base) rows = [] for i in results: # Fallback to the image name without the extension uri = i['name'].replace('.simg', '') # However the properties should include the uri if 'properties' in i: if 'uri' in i['properties']: uri = i['properties']['uri'] rows.append([i['id'], uri]) # Give the user back a uri i['uri'] = uri matches.append(i) bot.custom(prefix=" [drive://%s]" % self._base, message="\t\t[id]\t[uri]", color="PURPLE") bot.table(rows) return matches
def list_templates(self, name=None): '''list templates in the builder bundle library. If a name is provided, look it up Parameters ========== name: the name of a template to look up ''' configs = self._get_templates() rows = [] # DETAIL: The user wants to retrieve a particular configuration if name: matches = self._load_templates(name) bot.debug('Found %s matches for %s' % (len(matches), name)) for match in matches: print(json.dumps(match, indent=4, sort_keys=True)) # LISTING: If we don't have a specific name, just show all else: for config in configs['data']: rows.append([config['name']]) bot.table(rows)
def search_all(self, quiet=False): '''a "show all" search that doesn't require a query Parameters ========== quiet: if quiet is True, we only are using the function to return rows of results. ''' results = [] for obj in self.bucket.objects.all(): subsrc = obj.Object() # Metadata bug will capitalize all fields, workaround is to lowercase # https://github.com/boto/boto3/issues/1709 metadata = dict((k.lower(), v) for k, v in subsrc.metadata.items()) size = '' # MM-DD-YYYY datestr = "%s-%s-%s" % (obj.last_modified.month, obj.last_modified.day, obj.last_modified.year) if 'sizemb' in metadata: size = '%sMB' % metadata['sizemb'] results.append([obj.key, datestr, size]) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) if not quiet: bot.info("Containers") bot.table(results) return results