def get_method(method_namespace: str,
method_name: str,
method_version: int = None) -> "JSON":
"""
If method_version is None, get the latest snapshot
"""
if method_version is None:
list_methods = fapi.list_repository_methods(namespace=method_namespace,
name=method_name)
if list_methods.status_code != 200:
raise ValueError('Unable to list methods ' + ' - ' +
str(list_methods.json()))
methods = list_methods.json()
version = -1
for method in methods:
version = max(version, method['snapshotId'])
if version == -1:
raise ValueError(method_name + ' not found')
method_version = version
method_def = fapi.get_repository_method(method_namespace, method_name,
method_version)
if method_def.status_code != 200:
raise ValueError('Unable to fetch method {0}/{1}/{2} - {3}'.format(
method_namespace, method_name, method_version,
str(method_def.json())))
return method_def.json()
def main(argsv):
parser = argparse.ArgumentParser(
description='Download one or more methods from Broad Methods Repository'
)
parser.add_argument('-m',
'--method',
dest='method',
action='store',
required=True,
help=alto.METHOD_HELP)
args = parser.parse_args(argsv)
method_namespace, method_name, method_version = alto.fs_split(args.method)
list_methods = fapi.list_repository_methods(namespace=method_namespace,
name=method_name,
snapshotId=method_version)
if list_methods.status_code != 200:
raise ValueError('Unable to list methods ' + ' - ' +
str(list_methods.json))
methods = list_methods.json()
for method in methods:
wdl = fapi.get_repository_method(
method_namespace, method_name,
method['snapshotId']).json()['payload']
with open('{}_{}.wdl'.format(method_name, method['snapshotId']),
'w') as w:
w.write(wdl)
def validate_monitor_tasks(dependencies, args):
""" Validate that all entries in the supervisor are valid task configurations and
that all permissions requirements are satisfied.
"""
# Make a list of all task configurations needed to supervise
sup_configs = sorted(dependencies.keys())
try:
logging.info("Validating supervisor data...")
# Make an api call to list task configurations in the workspace
r = fapi.list_workspace_configs(args['project'], args['workspace'])
fapi._check_response_code(r, 200)
space_configs = r.json()
# Make a dict for easy lookup later
space_configs = {c["name"]: c for c in space_configs}
# Also make an api call to list methods you have view permissions for
r = fapi.list_repository_methods()
fapi._check_response_code(r, 200)
repo_methods = r.json()
## Put in a form that is more easily searchable: namespace/name:snapshot
repo_methods = {
m['namespace'] + '/' + m['name'] + ':' + str(m['snapshotId'])
for m in repo_methods if m['entityType'] == 'Workflow'
}
valid = True
for config in sup_configs:
# ensure config exists in the workspace
if config not in space_configs:
logging.error("No task configuration for " + config +
" found in " + args['project'] + "/" +
args['workspace'])
valid = False
else:
# Check access permissions for the referenced method
m = space_configs[config]['methodRepoMethod']
ref_method = m['methodNamespace'] + "/" + m[
'methodName'] + ":" + str(m['methodVersion'])
if ref_method not in repo_methods:
logging.error(
config +
" -- You don't have permisson to run the referenced method: "
+ ref_method)
valid = False
except Exception as e:
logging.error("Exception occurred while validating supervisor: " +
str(e))
raise
return False
return valid
def main(argv):
parser = argparse.ArgumentParser(
description='Remove methods from Broad Methods Repository.')
parser.add_argument(
'-m',
'--method',
dest='method',
action='store',
required=True,
help=
'Method takes the format of namespace/name/version. If only namespace is provided, delete all methods under that namespace. If both namespace and name are provided, delete all snapshots for that method. If namespace, name and version are provided, only delete the specific snapshot.'
)
args = parser.parse_args(argv)
fields = args.method.split('/')
if len(fields) == 0:
raise ValueError('No namespace specified!')
method_namespace = fields[0]
method_name = fields[1] if len(fields) > 0 else None
method_version = fields[2] if len(fields) > 1 else None
methods = fapi.list_repository_methods(namespace=method_namespace,
name=method_name).json()
if len(methods) == 0:
raise ValueError('No methods found')
if method_name is None: # delete all methods under specified namespace
for method in methods:
if method['namespace'] == method_namespace:
fapi.delete_repository_method(method['namespace'],
method['name'],
method['snapshotId'])
print(
f'Deleted {method["namespace"]}/{method["name"]}/{method["snapshotId"]}'
)
elif method_version is None: # delete all versions
for selected_method in methods:
if selected_method[
'namespace'] == method_namespace and selected_method[
'name'] == method_name:
fapi.delete_repository_method(selected_method['namespace'],
selected_method['name'],
selected_method['snapshotId'])
print(
f'Deleted {selected_method["namespace"]}/{selected_method["name"]}/{selected_method["snapshotId"]}'
)
else: # delete the specific version
selected_method = methods[0]
fapi.delete_repository_method(selected_method['namespace'],
selected_method['name'],
selected_method['snapshotId'])
print(
f'Deleted {selected_method["namespace"]}/{selected_method["name"]}/{selected_method["snapshotId"]}'
)
def main(argsv):
parser = argparse.ArgumentParser(
description='Remove one or more methods from Broad Methods Repository')
parser.add_argument('-m',
'--method',
dest='method',
action='store',
required=True,
help=alto.METHOD_HELP)
args = parser.parse_args(argsv)
method_namespace, method_name, method_version = alto.fs_split(args.method)
if method_namespace is None:
raise ValueError('No namespace specified')
if method_name == '':
method_name = None
methods = fapi.list_repository_methods(namespace=method_namespace,
name=method_name).json()
if len(methods) == 0:
raise ValueError('No methods found')
if method_name is None: # delete all methods under specified namespace
for method in methods:
if method['namespace'] == method_namespace:
fapi.delete_repository_method(method['namespace'],
method['name'],
method['snapshotId'])
print('Deleted {}/{}:{}'.format(method['namespace'],
method['name'],
method['snapshotId']))
else:
if method_version is None: # delete all versions
for selected_method in methods:
if selected_method[
'namespace'] == method_namespace and selected_method[
'name'] == method_name:
fapi.delete_repository_method(
selected_method['namespace'], selected_method['name'],
selected_method['snapshotId'])
print('Deleted {}/{}:{}'.format(
selected_method['namespace'], selected_method['name'],
selected_method['snapshotId']))
else:
selected_method = methods[0]
fapi.delete_repository_method(selected_method['namespace'],
selected_method['name'],
selected_method['snapshotId'])
print('Deleted {}/{}:{}'.format(selected_method['namespace'],
selected_method['name'],
selected_method['snapshotId']))
def get_firecloud_workflow(method_namespace: str,
method_name: str,
method_version: int = None) -> dict:
"""Locate a workflow using the method_namespace, method_name and method_version hierachy and return results in a dictionary.
Parameters
----------
method_namespace: `str`
The namespace that workflow belongs to (case-sensitive).
method_name: `str`
The workflow name (case-sensitive).
version: `int`, optional (default: None)
Snapshot ID. By default, the lastest snapshot recorded in Broad Methods Repository would be used.
Returns
-------
`dict` object.
A dictionary object containing the following entries:
'namespace': workflow namespace as recorded in Broad Methods Repository.
'name': workflow name as recorded in Broad Methods Repository.
'snapshotId': workflow version as recorded in Broad Methods Repository.
'url': URL to the WDL file.
'methodUri': Uniform Resource Identifier that is recognized by Terra platform.
Examples
--------
>>> results = get_firecloud_workflow('cumulus', 'cumulus')
"""
method_record = None
if method_version is not None:
method_def = fapi.get_repository_method(method_namespace, method_name,
method_version)
if method_def.status_code != 200:
raise ValueError(
f"Unable to fetch workflow {method_namespace}/{method_name}/{method_version} - {method_def.json()}!"
)
method_record = method_def.json()
else:
list_methods = fapi.list_repository_methods(namespace=method_namespace,
name=method_name)
if list_methods.status_code != 200:
raise ValueError(
f"Unable to list methods - {list_methods.json()}!")
methods = list_methods.json()
if len(methods) == 0:
raise ValueError(
f"Unable to locate workflow {method_namespace}/{method_name}!")
method_record = methods[0]
for method in methods[1:]:
if method_record["snapshotId"] < method["snapshotId"]:
method_record = method
results = {
"namespace":
method_record["namespace"],
"name":
method_record["name"],
"snapshotId":
method_record["snapshotId"],
"url":
f"https://api.firecloud.org/ga4gh/v1/tools/{method_record['namespace']}:{method_record['name']}/versions/{method_record['snapshotId']}/plain-WDL/descriptor",
"methodUri":
f"agora://{method_record['namespace']}/{method_record['name']}/{method_record['snapshotId']}"
}
return results
def test_list_repository_methods(self):
"""Test list_repository_methods()."""
r = fapi.list_repository_methods()
print(r.status_code, r.content)
self.assertEqual(r.status_code, 200)
def do_fc_run(method: str, workspace: str, wdl_inputs: Union[str, dict],
out_json: str, bucket_folder: str) -> str:
"""Run a FireCloud method.
Args:
method: method namespace/name/version. Version is optional
workspace: workspace namespace/name
wdl_inputs: WDL input JSON.
upload: Whether to upload inputs and convert local file paths to gs:// URLs.
bucket_folder: The folder under google bucket for uploading files.
Returns:
URL to check submission status
"""
inputs = kco.get_wdl_inputs(wdl_inputs)
method_namespace, method_name, method_version = kco.fs_split(method)
if method_version is None:
version = -1
list_methods = fapi.list_repository_methods(method_name)
if list_methods.status_code != 200:
raise ValueError('Unable to list methods ' + ' - ' +
str(list_methods.json))
methods = list_methods.json()
for method in methods:
if method['namespace'] == method_namespace:
version = max(version, method['snapshotId'])
if version == -1:
raise ValueError(method_name + ' not found')
method_version = version
root_entity = None
launch_entity = None
workspace_namespace, workspace_name, workspace_version = kco.fs_split(
workspace)
kco.get_or_create_workspace(workspace_namespace, workspace_name)
if out_json is not None:
kco.do_fc_upload(inputs, workspace, False, bucket_folder)
with open(out_json, 'w') as fout:
json.dump(inputs, fout)
config_namespace = method_namespace
config_name = method_name
method_body = {
'name': config_name,
'namespace': config_namespace,
'methodRepoMethod': {
'methodNamespace':
method_namespace,
'methodName':
method_name,
'methodVersion':
method_version,
'sourceRepo':
'agora',
'methodUri':
'agora://{0}/{1}/{2}'.format(method_namespace, method_name,
method_version)
},
'rootEntityType': root_entity,
'prerequisites': {},
'inputs': convert_inputs(inputs),
'outputs': {},
'methodConfigVersion': 1,
'deleted': False
}
config_exists = fapi.get_workspace_config(workspace_namespace,
workspace_name, config_namespace,
config_name)
if config_exists.status_code == 200:
config_submission = fapi.update_workspace_config(
workspace_namespace, workspace_name, config_namespace, config_name,
method_body)
if config_submission.status_code != 200:
raise ValueError('Unable to update workspace config. Response: ' +
str(config_submission.status_code))
else:
config_submission = fapi.create_workspace_config(
workspace_namespace, workspace_name, method_body)
if config_submission.status_code != 201:
raise ValueError('Unable to create workspace config - ' +
str(config_submission.json()))
launch_submission = fapi.create_submission(workspace_namespace,
workspace_name,
config_namespace, config_name,
launch_entity, root_entity, "")
if launch_submission.status_code == 201:
submission_id = launch_submission.json()['submissionId']
url = 'https://portal.firecloud.org/#workspaces/{}/{}/monitor/{}'.format(
workspace_namespace, workspace_name, submission_id)
return url
else:
raise ValueError('Unable to launch submission - ' +
str(launch_submission.json()))
def do_fc_inputs(method, out, config):
method_namespace, method_name, method_version = kco.fs_split(method)
if method_namespace is None:
raise ValueError(
'Method should be specified as namespace/name (e.g. regev/drop-seq)'
)
if method_version is None:
version = -1
list_methods = fapi.list_repository_methods(method_name)
if list_methods.status_code != 200:
raise ValueError('Unable to list methods ' + ' - ' +
str(list_methods.json))
methods = list_methods.json()
for method in methods:
if method['namespace'] == method_namespace:
version = max(version, method['snapshotId'])
if version == -1:
raise ValueError(method_name + ' not found')
method_version = version
payload = fapi.get_repository_method(method_namespace, method_name,
method_version).json()['payload']
tmp_wdl_file = tempfile.mkstemp(suffix='.wdl')[1]
with open(tmp_wdl_file, 'w') as w:
w.write(payload)
repo_config = None
if config is not None:
config_namespace, config_name, config_version = kco.fs_split(
config)
if config_version is not None:
repo_config = fapi.get_repository_config(
config_namespace, config_name, config_version).json()
else:
repo_configs = fapi.list_repository_configs().json()
for config in repo_configs:
if config['namespace'] == config_namespace and config[
'name'] == config_name and config['method'][
'name'] == method_name and config['method'][
'namespace'] == method_namespace:
repo_config = config
break
if repo_config is not None:
repo_config = fapi.get_repository_config(
repo_config['namespace'], repo_config['name'],
repo_config['snapshotId']).json()
jar = pkg_resources.resource_filename('kco', 'womtool-33.1.jar')
tmp_json_file = tempfile.mkstemp(suffix='.json')[1]
with open(tmp_json_file, 'w') as tmp_json_out:
subprocess.check_call(
['java', '-jar', jar, 'inputs', tmp_wdl_file],
stdout=tmp_json_out)
with open(tmp_json_file, 'r') as tmp_json_in:
inputs = json.loads(tmp_json_in.read())
if repo_config is not None:
repo_config = json.loads(repo_config['payload'])['inputs']
for key in repo_config:
value = repo_config[key]
if inputs.get(key) is not None and value != '':
inputs[key] = value
if out is None:
out = method_name + '_inputs.json'
if not out.lower().endswith('.json'):
out = out + '.json'
with open(out, 'w') as json_out:
json.dump(inputs, json_out, indent=0)
os.remove(tmp_json_file)
os.remove(tmp_wdl_file)
SEL_WORKSPACE = 'selection-sim'
#dir(fapi)
#help(fapi)
z = fapi.list_workspace_configs(namespace=SEL_NAMESPACE,
workspace=SEL_WORKSPACE,
allRepos=True).json()
print(z)
z = fapi.get_workspace_config(workspace=SEL_WORKSPACE,
namespace=SEL_NAMESPACE,
config='dockstore-tool-cosi2',
cnamespace=SEL_NAMESPACE)
print('CONFIG_IS', z, z.json())
z = fapi.list_repository_methods(namespace=SEL_NAMESPACE,
name='test-cosi2-method-01').json()
print('METHODS LIST BEF', z)
z = fapi.update_repository_method(
namespace=SEL_NAMESPACE,
method='test-cosi2-method-01',
synopsis='run cosi2',
wdl='/data/ilya-work/proj/dockstore-tool-cosi2/Dockstore.wdl')
print('UPDATE IS', z, z.json())
new_method = z.json()
z = fapi.list_repository_methods(namespace=SEL_NAMESPACE,
name='test-cosi2-method-01').json()
print('METHODS LIST AFT', z)
z = fapi.get_config_template(namespace=SEL_NAMESPACE,
