def tool_shed_client(ctx=None, **kwds): if toolshed is None: raise Exception(BIOBLEND_UNAVAILABLE) read_only = kwds.get("read_only", False) shed_target = kwds.get("shed_target") global_config = getattr(ctx, "global_config", {}) if global_config and "sheds" in global_config: sheds_config = global_config["sheds"] shed_config = sheds_config.get(shed_target, {}) else: shed_config = {} def prop(key): return kwds.get("shed_%s" % key, None) or shed_config.get(key, None) url = _tool_shed_url(kwds) if read_only: key = None email = None password = None else: key = prop("key") email = prop("email") password = prop("password") tsi = toolshed.ToolShedInstance(url=url, key=key, email=email, password=password) return tsi
def __init__(self): """ Initializes the Tools class and connects to a toolshed instance """ self.gi = create_instance() # create a galaxy instance. self.ts = toolshed.ToolShedInstance( url='https://usegalaxy.eu/') # create a toolshed instance
def update_file(fn, owner=None, name=None, without=False): with open(fn + '.lock', 'r') as handle: locked = yaml.safe_load(handle) # Update any locked tools. for tool in locked['tools']: # If without, then if it is lacking, we should exec. logging.debug("Examining {owner}/{name}".format(**tool)) if without: if 'revisions' in tool and not len(tool.get('revisions', [])) == 0: continue if not without and owner and tool['owner'] != owner: continue if not without and name and tool['name'] != name: continue logging.info("Fetching updates for {owner}/{name}".format(**tool)) if 'tool_shed_url' in tool: if not tool['tool_shed_url'] in ts: ts[tool['tool_shed_url']] = toolshed.ToolShedInstance( url='https://' + tool['tool_shed_url']) logging.info("Using toolshed {}".format(tool['tool_shed_url'])) toolshed = ts[tool['tool_shed_url']] else: toolshed = ts[main] try: revs = toolshed.repositories.get_ordered_installable_revisions( tool['name'], tool['owner']) except Exception as e: print(e) continue logging.debug('TS revisions: %s' % ','.join(revs)) latest_rev = revs[-1] if latest_rev in tool.get('revisions', []): # The rev is already known, don't add again. continue logging.info("Found newer revision of {owner}/{name} ({rev})".format( rev=latest_rev, **tool)) # Get latest rev, if not already added, add it. if 'revisions' not in tool: tool['revisions'] = [] if latest_rev not in tool['revisions']: # TS doesn't support utf8 and we don't want to either. tool['revisions'].append(str(latest_rev)) tool['revisions'] = sorted(list(set(tool['revisions']))) with open(fn + '.lock', 'w') as handle: yaml.dump(locked, handle, default_flow_style=False)
def repos(tool_shed_url, name_filter=None, owner=None): ts = toolshed.ToolShedInstance(url=TOOLSHED) repos = ts.repositories.get_repositories() if owner: repos = [r for r in repos if r["owner"] == owner] if name_filter: pattern = re.compile(name_filter) repos = [r for r in repos if pattern.match(r["name"])] return repos
def shedLoad(self): """ use bioblend to create new repository or update existing """ if os.path.exists(self.tlog): sto = open(self.tlog, "a") else: sto = open(self.tlog, "w") ts = toolshed.ToolShedInstance( url=self.args.toolshed_url, key=self.args.toolshed_api_key, verify=False, ) repos = ts.repositories.get_repositories() rnames = [x.get("name", "?") for x in repos] rids = [x.get("id", "?") for x in repos] tfcat = "ToolFactory generated tools" if self.tool_name not in rnames: tscat = ts.categories.get_categories() cnames = [x.get("name", "?").strip() for x in tscat] cids = [x.get("id", "?") for x in tscat] catID = None if tfcat.strip() in cnames: ci = cnames.index(tfcat) catID = cids[ci] res = ts.repositories.create_repository( name=self.args.tool_name, synopsis="Synopsis:%s" % self.args.tool_desc, description=self.args.tool_desc, type="unrestricted", remote_repository_url=self.args.toolshed_url, homepage_url=None, category_ids=catID, ) tid = res.get("id", None) sto.write( f"#create_repository {self.args.tool_name} tid={tid} res={res}\n" ) else: i = rnames.index(self.tool_name) tid = rids[i] try: res = ts.repositories.update_repository( id=tid, tar_ball_path=self.newtarpath, commit_message=None) sto.write(f"#update res id {id} ={res}\n") except ConnectionError: sto.write( "####### Is the toolshed running and the API key correct? Bioblend shed upload failed\n" ) sto.close()
def retrieve_changeset_revision(ts_url, name, owner): ts = toolshed.ToolShedInstance(url=ts_url) ts_repositories = ts.repositories.get_repositories() ts_id = None for repo in ts_repositories: if str(repo['name']) == name and str(repo['owner']) == owner: ts_id = repo['id'] if ts_id == None: string = "No repository found for " + name + " (" + owner + ")" string += " in toolshed at " + ts_url raise ValueError(string) return ts.repositories.show_repository_revision( ts_id)['changeset_revision']
def test_shed(self): shed_url = os.environ.get("TEST_TOOL_SHED_URL", "http://localhost:9009") shed_api_key = os.environ.get("TEST_TOOL_SHED_API_KEY") tsi = toolshed.ToolShedInstance(shed_url, key=shed_api_key) owner = username(tsi) name = "planemotestrepo%d" % random.randint(0, 1000000) with self._isolate(): shed_yml_contents = SHED_TEMPLATE.safe_substitute( owner=owner, name=name, ) open(".shed.yml", "w").write(shed_yml_contents) init_cmd = [ "shed_create", "--shed_key", shed_api_key, "--shed_target", shed_url ] self._check_exit_code(init_cmd)
def test_shed(self): shed_url = os.environ.get("TEST_TOOL_SHED_URL", "http://localhost:9009") shed_api_key = os.environ.get("TEST_TOOL_SHED_API_KEY") tsi = toolshed.ToolShedInstance(shed_url, key=shed_api_key) owner = username(tsi) name = "planemotestrepo%d" % random.randint(0, 1000000) with self._isolate(): shed_yml_contents = SHED_TEMPLATE.safe_substitute( owner=owner, name=name, ) io.write_file(".shed.yml", shed_yml_contents) test_path = os.path.join(TEST_DIR, "tool_dependencies_good_1.xml") contents = open(test_path).read() io.write_file("tool_dependencies.xml", contents) init_cmd = [ "shed_create", "--shed_key", shed_api_key, "--shed_target", shed_url ] self._check_exit_code(init_cmd) contents_dict = yaml.load(open(".shed.yml", "r")) contents_dict["description"] = "Update test repository." io.write_file(".shed.yml", yaml.dump(contents_dict)) update_cmd = [ "shed_update", "--shed_key", shed_api_key, "--shed_target", shed_url ] self._check_exit_code(update_cmd) upload_cmd = [ "shed_upload", "--shed_key", shed_api_key, "--shed_target", shed_url ] self._check_exit_code(upload_cmd) download_cmd = [ "shed_download", "--shed_target", shed_url, "--destination", "shed_download.tar.gz" ] self._check_exit_code(download_cmd)
def update_file(add_to_list, infile, outfile): with open(infile, "r") as f: data = yaml.safe_load(f) tool_list = data[TOOLS] for tool in tool_list: print(f"Getting latest revision for {tool[NAME]}") validate(tool) url = tool[SHED] if url in tool_sheds: ts = tool_sheds[url] else: ts = toolshed.ToolShedInstance(url) tool_sheds[url] = ts revs = ts.repositories.get_ordered_installable_revisions(tool[NAME], tool[OWNER]) if revs and len(revs) > 0: add_to_list(tool, revs[-1]) data = { "tools": tool_list } with open(outfile, "w") as f: yaml.dump(data, f, default_flow_style=False)
import yaml import os import glob import copy import argparse import logging from bioblend import toolshed ts = toolshed.ToolShedInstance(url='https://toolshed.g2.bx.psu.edu') def update_file(fn, owner=None, name=None, without=False): with open(fn + '.lock', 'r') as handle: locked = yaml.safe_load(handle) # Update any locked tools. for tool in locked['tools']: # If without, then if it is lacking, we should exec. logging.debug("Examining {owner}/{name}".format(**tool)) if without: if 'revisions' in tool and not len(tool.get('revisions', [])) == 0: continue if not without and owner and tool['owner'] != owner: continue if not without and name and tool['name'] != name: continue
python .github/scripts/update_tools.py [append|replace] ./production/anvil/tools.yml /path/to/write.yml """ DEFAULT_TOOLSHED = 'https://toolshed.g2.bx.psu.edu' # Common keys into the tools dict. Defined solely so our IDE can do completions # and I don't consistently misspell revisisions or have to remember if it is # toolshed_url or tool_shed_url NAME = 'name' OWNER = 'owner' TOOLS = 'tools' SHED = 'tool_shed_url' REVISIONS = 'revisions' # The toolsheds that we have already connected to. tool_sheds = { DEFAULT_TOOLSHED: toolshed.ToolShedInstance(DEFAULT_TOOLSHED) } def validate(tool): """Ensure the tool has the fields we need so we don't need to check later.""" if SHED not in tool: tool[SHED] = DEFAULT_TOOLSHED if REVISIONS not in tool: tool[REVISIONS] = [] def append(tool, revision): if revision not in tool[REVISIONS]: tool[REVISIONS].append(revision) def replace(tool, revision): tool[REVISIONS] = [ revision ]
import yaml import os import glob import copy import argparse import logging # From https://raw.githubusercontent.com/usegalaxy-eu/usegalaxy-eu-tools/master/scripts/update-tool.py from bioblend import toolshed ts = dict() main = 'toolshed.g2.bx.psu.edu' test = 'testtoolshed.g2.bx.psu.edu' ts[main] = toolshed.ToolShedInstance(url='https://' + main) ts[test] = toolshed.ToolShedInstance(url='https://' + test) def update_file(fn, owner=None, name=None, without=False): with open(fn + '.lock', 'r') as handle: locked = yaml.safe_load(handle) # Update any locked tools. for tool in locked['tools']: # If without, then if it is lacking, we should exec. logging.debug("Examining {owner}/{name}".format(**tool)) if without: if 'revisions' in tool and not len(tool.get('revisions', [])) == 0: continue
def get_toolshed_instance(instance_name=None): conf = get_instance(instance_name=instance_name) return toolshed.ToolShedInstance(conf['url'], conf['key'])
def search_toolshed(tool_shed, query_string, gi=None, long_listing_format=False): """ Search toolshed and print resulting matches Arguments: tool_shed (str): URL for tool shed to search query_string (str): text to use as query gi (bioblend.galaxy.GalaxyInstance): Galaxy instance long_listing_format (boolean): if True then use a long listing format when reporting items """ # Get a toolshed instance tool_shed_url = normalise_toolshed_url(tool_shed) shed = toolshed.ToolShedInstance(tool_shed_url) print("Searching %s" % tool_shed_url) # Remove wildcards from start and end of query string shed_query_string = query_string.strip("*") # Query the toolshed repo_client = toolshed.repositories.ToolShedRepositoryClient(shed) try: search_result = repo_client.search_repositories( shed_query_string, page_size=SEARCH_PAGE_SIZE) except BioblendConnectionError as connection_error: # Handle error logger.warning("Error from Galaxy API: %s" % connection_error) return connection_error.status_code # Filter on name hits = [ r for r in search_result['hits'] if fnmatch(r["repository"]["name"].lower(), query_string) ] # Deal with the results nhits = len(hits) if nhits == 0: print("No matching repositories found") return 0 # Get additional details for each repo repositories = list() for hit in hits: # Get the repository details repo = hit['repository'] name = repo['name'] owner = repo['repo_owner_username'] description = to_ascii(repo['description']).strip() # Get installable revisions installable_revisions = list() for revision in \ repo_client.get_ordered_installable_revisions(name,owner): # Get details for each revision revision_info = \ repo_client.get_repository_revision_install_info( name, owner, revision) # Returns a 3 element list, only want details # from the last one # See https://bioblend.readthedocs.io/en/latest/api_docs/toolshed/all.html#bioblend.toolshed.repositories.ToolShedRepositoryClient.get_repository_revision_install_info revision_info = revision_info[2] version = revision_info[name][3] installable_revisions.append( dict(revision=revision, version=version, info=revision_info)) # Sort the installable revisions on version number installable_revisions = sorted(installable_revisions, key=lambda r: int(r['version']), reverse=True) # Sort repo details repositories.append( dict(name=name, owner=owner, description=description, revisions=installable_revisions)) # Get list of installed tool repositories if gi is not None: # Strip protocol from tool shed URL tool_shed = tool_shed_url for proc in ('http://', 'https://'): if tool_shed.startswith(proc): tool_shed = tool_shed[len(proc):] # Strip trailing slash tool_shed = tool_shed.rstrip('/') # Restrict repos to this tool shed installed_repos = [ r for r in get_repositories(gi) if r.tool_shed == tool_shed ] else: installed_repos = [] # Print the results print("") output = Reporter() for repository in repositories: # Get the repository details name = repository['name'] owner = repository['owner'] description = repository['description'] # Iterate over revisions for revision in repository['revisions']: changeset = revision['revision'] version = revision['version'] # Look to see it's installed installed = bool([ r for r in installed_repos if (r.name == name and r.owner == owner and changeset in [rv.changeset_revision for rv in r.revisions()]) ]) if installed: status = "*" else: status = " " url = os.path.join(tool_shed_url, "view", owner, name, changeset) # Print details if not long_listing_format: display_items = [owner, name, f"{version}:{changeset}", status] output.append(display_items) else: output.append(("Name", name)) output.append(("Owner", owner)) output.append(("Revision", f"{version}:{changeset}")) output.append(("Description", description)) output.append(("URL", url)) if gi is not None: if installed: output.append(("Installed", "yes")) else: output.append(("Installed", "no")) output.append(("", )) if not long_listing_format: output.report(prefix=" ") else: output.report(delimiter=": ") print("\n%d repositor%s found" % (nhits, ('y' if nhits == 1 else 'ies'))) # Finished return 0