def tool_shed_client(ctx=None, **kwds):
if toolshed is None:
raise Exception(BIOBLEND_UNAVAILABLE)
read_only = kwds.get("read_only", False)
shed_target = kwds.get("shed_target")
global_config = getattr(ctx, "global_config", {})
if global_config and "sheds" in global_config:
sheds_config = global_config["sheds"]
shed_config = sheds_config.get(shed_target, {})
else:
shed_config = {}
def prop(key):
return kwds.get("shed_%s" % key, None) or shed_config.get(key, None)
url = _tool_shed_url(kwds)
if read_only:
key = None
email = None
password = None
else:
key = prop("key")
email = prop("email")
password = prop("password")
tsi = toolshed.ToolShedInstance(url=url,
key=key,
email=email,
password=password)
return tsi
def __init__(self):
"""
Initializes the Tools class and connects to a toolshed instance
"""
self.gi = create_instance() # create a galaxy instance.
self.ts = toolshed.ToolShedInstance(
url='https://usegalaxy.eu/') # create a toolshed instance
def update_file(fn, owner=None, name=None, without=False):
with open(fn + '.lock', 'r') as handle:
locked = yaml.safe_load(handle)
# Update any locked tools.
for tool in locked['tools']:
# If without, then if it is lacking, we should exec.
logging.debug("Examining {owner}/{name}".format(**tool))
if without:
if 'revisions' in tool and not len(tool.get('revisions', [])) == 0:
continue
if not without and owner and tool['owner'] != owner:
continue
if not without and name and tool['name'] != name:
continue
logging.info("Fetching updates for {owner}/{name}".format(**tool))
if 'tool_shed_url' in tool:
if not tool['tool_shed_url'] in ts:
ts[tool['tool_shed_url']] = toolshed.ToolShedInstance(
url='https://' + tool['tool_shed_url'])
logging.info("Using toolshed {}".format(tool['tool_shed_url']))
toolshed = ts[tool['tool_shed_url']]
else:
toolshed = ts[main]
try:
revs = toolshed.repositories.get_ordered_installable_revisions(
tool['name'], tool['owner'])
except Exception as e:
print(e)
continue
logging.debug('TS revisions: %s' % ','.join(revs))
latest_rev = revs[-1]
if latest_rev in tool.get('revisions', []):
# The rev is already known, don't add again.
continue
logging.info("Found newer revision of {owner}/{name} ({rev})".format(
rev=latest_rev, **tool))
# Get latest rev, if not already added, add it.
if 'revisions' not in tool:
tool['revisions'] = []
if latest_rev not in tool['revisions']:
# TS doesn't support utf8 and we don't want to either.
tool['revisions'].append(str(latest_rev))
tool['revisions'] = sorted(list(set(tool['revisions'])))
with open(fn + '.lock', 'w') as handle:
yaml.dump(locked, handle, default_flow_style=False)
def repos(tool_shed_url, name_filter=None, owner=None):
ts = toolshed.ToolShedInstance(url=TOOLSHED)
repos = ts.repositories.get_repositories()
if owner:
repos = [r for r in repos if r["owner"] == owner]
if name_filter:
pattern = re.compile(name_filter)
repos = [r for r in repos if pattern.match(r["name"])]
return repos
def shedLoad(self):
"""
use bioblend to create new repository
or update existing
"""
if os.path.exists(self.tlog):
sto = open(self.tlog, "a")
else:
sto = open(self.tlog, "w")
ts = toolshed.ToolShedInstance(
url=self.args.toolshed_url,
key=self.args.toolshed_api_key,
verify=False,
)
repos = ts.repositories.get_repositories()
rnames = [x.get("name", "?") for x in repos]
rids = [x.get("id", "?") for x in repos]
tfcat = "ToolFactory generated tools"
if self.tool_name not in rnames:
tscat = ts.categories.get_categories()
cnames = [x.get("name", "?").strip() for x in tscat]
cids = [x.get("id", "?") for x in tscat]
catID = None
if tfcat.strip() in cnames:
ci = cnames.index(tfcat)
catID = cids[ci]
res = ts.repositories.create_repository(
name=self.args.tool_name,
synopsis="Synopsis:%s" % self.args.tool_desc,
description=self.args.tool_desc,
type="unrestricted",
remote_repository_url=self.args.toolshed_url,
homepage_url=None,
category_ids=catID,
)
tid = res.get("id", None)
sto.write(
f"#create_repository {self.args.tool_name} tid={tid} res={res}\n"
)
else:
i = rnames.index(self.tool_name)
tid = rids[i]
try:
res = ts.repositories.update_repository(
id=tid, tar_ball_path=self.newtarpath, commit_message=None)
sto.write(f"#update res id {id} ={res}\n")
except ConnectionError:
sto.write(
"####### Is the toolshed running and the API key correct? Bioblend shed upload failed\n"
)
sto.close()
def retrieve_changeset_revision(ts_url, name, owner):
ts = toolshed.ToolShedInstance(url=ts_url)
ts_repositories = ts.repositories.get_repositories()
ts_id = None
for repo in ts_repositories:
if str(repo['name']) == name and str(repo['owner']) == owner:
ts_id = repo['id']
if ts_id == None:
string = "No repository found for " + name + " (" + owner + ")"
string += " in toolshed at " + ts_url
raise ValueError(string)
return ts.repositories.show_repository_revision(
ts_id)['changeset_revision']
def test_shed(self):
shed_url = os.environ.get("TEST_TOOL_SHED_URL",
"http://localhost:9009")
shed_api_key = os.environ.get("TEST_TOOL_SHED_API_KEY")
tsi = toolshed.ToolShedInstance(shed_url, key=shed_api_key)
owner = username(tsi)
name = "planemotestrepo%d" % random.randint(0, 1000000)
with self._isolate():
shed_yml_contents = SHED_TEMPLATE.safe_substitute(
owner=owner,
name=name,
)
open(".shed.yml", "w").write(shed_yml_contents)
init_cmd = [
"shed_create", "--shed_key", shed_api_key, "--shed_target",
shed_url
]
self._check_exit_code(init_cmd)
def test_shed(self):
shed_url = os.environ.get("TEST_TOOL_SHED_URL",
"http://localhost:9009")
shed_api_key = os.environ.get("TEST_TOOL_SHED_API_KEY")
tsi = toolshed.ToolShedInstance(shed_url, key=shed_api_key)
owner = username(tsi)
name = "planemotestrepo%d" % random.randint(0, 1000000)
with self._isolate():
shed_yml_contents = SHED_TEMPLATE.safe_substitute(
owner=owner,
name=name,
)
io.write_file(".shed.yml", shed_yml_contents)
test_path = os.path.join(TEST_DIR, "tool_dependencies_good_1.xml")
contents = open(test_path).read()
io.write_file("tool_dependencies.xml", contents)
init_cmd = [
"shed_create",
"--shed_key", shed_api_key,
"--shed_target", shed_url
]
self._check_exit_code(init_cmd)
contents_dict = yaml.load(open(".shed.yml", "r"))
contents_dict["description"] = "Update test repository."
io.write_file(".shed.yml", yaml.dump(contents_dict))
update_cmd = [
"shed_update",
"--shed_key", shed_api_key,
"--shed_target", shed_url
]
self._check_exit_code(update_cmd)
upload_cmd = [
"shed_upload",
"--shed_key", shed_api_key,
"--shed_target", shed_url
]
self._check_exit_code(upload_cmd)
download_cmd = [
"shed_download",
"--shed_target", shed_url,
"--destination", "shed_download.tar.gz"
]
self._check_exit_code(download_cmd)
def update_file(add_to_list, infile, outfile):
with open(infile, "r") as f:
data = yaml.safe_load(f)
tool_list = data[TOOLS]
for tool in tool_list:
print(f"Getting latest revision for {tool[NAME]}")
validate(tool)
url = tool[SHED]
if url in tool_sheds:
ts = tool_sheds[url]
else:
ts = toolshed.ToolShedInstance(url)
tool_sheds[url] = ts
revs = ts.repositories.get_ordered_installable_revisions(tool[NAME], tool[OWNER])
if revs and len(revs) > 0:
add_to_list(tool, revs[-1])
data = { "tools": tool_list }
with open(outfile, "w") as f:
yaml.dump(data, f, default_flow_style=False)
import yaml
import os
import glob
import copy
import argparse
import logging
from bioblend import toolshed
ts = toolshed.ToolShedInstance(url='https://toolshed.g2.bx.psu.edu')
def update_file(fn, owner=None, name=None, without=False):
with open(fn + '.lock', 'r') as handle:
locked = yaml.safe_load(handle)
# Update any locked tools.
for tool in locked['tools']:
# If without, then if it is lacking, we should exec.
logging.debug("Examining {owner}/{name}".format(**tool))
if without:
if 'revisions' in tool and not len(tool.get('revisions', [])) == 0:
continue
if not without and owner and tool['owner'] != owner:
continue
if not without and name and tool['name'] != name:
continue
python .github/scripts/update_tools.py [append|replace] ./production/anvil/tools.yml /path/to/write.yml
"""
DEFAULT_TOOLSHED = 'https://toolshed.g2.bx.psu.edu'
# Common keys into the tools dict. Defined solely so our IDE can do completions
# and I don't consistently misspell revisisions or have to remember if it is
# toolshed_url or tool_shed_url
NAME = 'name'
OWNER = 'owner'
TOOLS = 'tools'
SHED = 'tool_shed_url'
REVISIONS = 'revisions'
# The toolsheds that we have already connected to.
tool_sheds = { DEFAULT_TOOLSHED: toolshed.ToolShedInstance(DEFAULT_TOOLSHED) }
def validate(tool):
"""Ensure the tool has the fields we need so we don't need to check later."""
if SHED not in tool:
tool[SHED] = DEFAULT_TOOLSHED
if REVISIONS not in tool:
tool[REVISIONS] = []
def append(tool, revision):
if revision not in tool[REVISIONS]:
tool[REVISIONS].append(revision)
def replace(tool, revision):
tool[REVISIONS] = [ revision ]
import yaml
import os
import glob
import copy
import argparse
import logging
# From https://raw.githubusercontent.com/usegalaxy-eu/usegalaxy-eu-tools/master/scripts/update-tool.py
from bioblend import toolshed
ts = dict()
main = 'toolshed.g2.bx.psu.edu'
test = 'testtoolshed.g2.bx.psu.edu'
ts[main] = toolshed.ToolShedInstance(url='https://' + main)
ts[test] = toolshed.ToolShedInstance(url='https://' + test)
def update_file(fn, owner=None, name=None, without=False):
with open(fn + '.lock', 'r') as handle:
locked = yaml.safe_load(handle)
# Update any locked tools.
for tool in locked['tools']:
# If without, then if it is lacking, we should exec.
logging.debug("Examining {owner}/{name}".format(**tool))
if without:
if 'revisions' in tool and not len(tool.get('revisions', [])) == 0:
continue
def get_toolshed_instance(instance_name=None):
conf = get_instance(instance_name=instance_name)
return toolshed.ToolShedInstance(conf['url'], conf['key'])
def search_toolshed(tool_shed,
query_string,
gi=None,
long_listing_format=False):
"""
Search toolshed and print resulting matches
Arguments:
tool_shed (str): URL for tool shed to search
query_string (str): text to use as query
gi (bioblend.galaxy.GalaxyInstance): Galaxy instance
long_listing_format (boolean): if True then use a
long listing format when reporting items
"""
# Get a toolshed instance
tool_shed_url = normalise_toolshed_url(tool_shed)
shed = toolshed.ToolShedInstance(tool_shed_url)
print("Searching %s" % tool_shed_url)
# Remove wildcards from start and end of query string
shed_query_string = query_string.strip("*")
# Query the toolshed
repo_client = toolshed.repositories.ToolShedRepositoryClient(shed)
try:
search_result = repo_client.search_repositories(
shed_query_string, page_size=SEARCH_PAGE_SIZE)
except BioblendConnectionError as connection_error:
# Handle error
logger.warning("Error from Galaxy API: %s" % connection_error)
return connection_error.status_code
# Filter on name
hits = [
r for r in search_result['hits']
if fnmatch(r["repository"]["name"].lower(), query_string)
]
# Deal with the results
nhits = len(hits)
if nhits == 0:
print("No matching repositories found")
return 0
# Get additional details for each repo
repositories = list()
for hit in hits:
# Get the repository details
repo = hit['repository']
name = repo['name']
owner = repo['repo_owner_username']
description = to_ascii(repo['description']).strip()
# Get installable revisions
installable_revisions = list()
for revision in \
repo_client.get_ordered_installable_revisions(name,owner):
# Get details for each revision
revision_info = \
repo_client.get_repository_revision_install_info(
name,
owner,
revision)
# Returns a 3 element list, only want details
# from the last one
# See https://bioblend.readthedocs.io/en/latest/api_docs/toolshed/all.html#bioblend.toolshed.repositories.ToolShedRepositoryClient.get_repository_revision_install_info
revision_info = revision_info[2]
version = revision_info[name][3]
installable_revisions.append(
dict(revision=revision, version=version, info=revision_info))
# Sort the installable revisions on version number
installable_revisions = sorted(installable_revisions,
key=lambda r: int(r['version']),
reverse=True)
# Sort repo details
repositories.append(
dict(name=name,
owner=owner,
description=description,
revisions=installable_revisions))
# Get list of installed tool repositories
if gi is not None:
# Strip protocol from tool shed URL
tool_shed = tool_shed_url
for proc in ('http://', 'https://'):
if tool_shed.startswith(proc):
tool_shed = tool_shed[len(proc):]
# Strip trailing slash
tool_shed = tool_shed.rstrip('/')
# Restrict repos to this tool shed
installed_repos = [
r for r in get_repositories(gi) if r.tool_shed == tool_shed
]
else:
installed_repos = []
# Print the results
print("")
output = Reporter()
for repository in repositories:
# Get the repository details
name = repository['name']
owner = repository['owner']
description = repository['description']
# Iterate over revisions
for revision in repository['revisions']:
changeset = revision['revision']
version = revision['version']
# Look to see it's installed
installed = bool([
r for r in installed_repos
if (r.name == name and r.owner == owner and changeset in
[rv.changeset_revision for rv in r.revisions()])
])
if installed:
status = "*"
else:
status = " "
url = os.path.join(tool_shed_url, "view", owner, name, changeset)
# Print details
if not long_listing_format:
display_items = [owner, name, f"{version}:{changeset}", status]
output.append(display_items)
else:
output.append(("Name", name))
output.append(("Owner", owner))
output.append(("Revision", f"{version}:{changeset}"))
output.append(("Description", description))
output.append(("URL", url))
if gi is not None:
if installed:
output.append(("Installed", "yes"))
else:
output.append(("Installed", "no"))
output.append(("", ))
if not long_listing_format:
output.report(prefix=" ")
else:
output.report(delimiter=": ")
print("\n%d repositor%s found" % (nhits, ('y' if nhits == 1 else 'ies')))
# Finished
return 0
