def setUpClass(cls):
'''Set up FireCloud etc to run tests'''
print("\nStarting high-level CLI tests ...\n", file=sys.stderr)
fiss_verbosity = os.environ.get("FISS_TEST_VERBOSITY", None)
if fiss_verbosity == None:
fiss_verbosity = 0
fcconfig = fccore.config_get_all()
fcconfig.set_verbosity(fiss_verbosity)
cls.project = fcconfig.project
if not cls.project:
raise ValueError("Your configuration must define a FireCloud project")
# Set up a temp workspace for duration of tests. And in case a previous
# test failed, we attempt to unlock & delete before creating anew
cls.workspace = getuser() + '_FISS_TEST'
ret = call_func("space_exists", "-p", cls.project, "-w", cls.workspace)
if ret == True and os.environ.get("REUSE_SPACE", None):
return
print("\tCreating test workspace ...\n", file=sys.stderr)
r = fapi.unlock_workspace(cls.project, cls.workspace)
r = fapi.delete_workspace(cls.project, cls.workspace)
r = fapi.create_workspace(cls.project, cls.workspace)
fapi._check_response_code(r, 201)
def __get_entities(self, etype):
"""Helper to get entities for a given type."""
r = fapi.get_entities(self.namespace, self.name,
etype, self.api_url)
fapi._check_response_code(r, 200)
return [Entity(e['entityType'], e['name'], e['attributes'])
for e in r.json()]
def main():
args = parse_args()
resp = fapi.get_entities_with_type(args.workspace_namespace,
args.workspace_name)
fapi._check_response_code(resp, 200)
entities_by_type = {}
for entity in resp.json():
entity_type = entity['entityType']
if entity_type not in entities_by_type:
entities_by_type[entity_type] = []
entities_by_type[entity_type].append(entity['name'])
# Entities are sorted by type: participant, participant_set, sample,
# sample_set. FireCloud complains if we delete a participant before deleting
# associated participant_set/sample/sample_set.
for entity_type in FC_ENTITY_TYPES:
if entity_type in entities_by_type:
entities = entities_by_type[entity_type]
delete_entity_type(args, entity_type, entities)
del entities_by_type[entity_type]
# Delete the remaining entities where order does not matter.
for entity_type, entities in entities_by_type.iteritems():
if entity_type not in FC_ENTITY_TYPES:
delete_entity_type(args, entity_type, entities)
def batch_load(project,
workspace,
headerline,
entity_data,
chunk_size=500,
model='flexible'):
""" Submit a large number of entity updates in batches of chunk_size """
# Parse the entity type from the first cell, e.g. "entity:sample_id"
# First check that the header is valid
# if not valid_headerline(headerline, model):
# print("Invalid loadfile header:\n" + headerline)
# return 1
update_type = "membership" if headerline.startswith(
"membership") else "entity"
etype = headerline.split('\t')[0].split(':')[1].replace("_id", "")
# Split entity_data into chunks
total = int(len(entity_data) / chunk_size) + 1
batch = 0
for i in range(0, len(entity_data), chunk_size):
batch += 1
print("Updating {0} {1}s {2}-{3}, batch {4}/{5}".format(
etype, update_type, i + 1, min(i + chunk_size, len(entity_data)),
batch, total))
this_data = headerline + '\n' + '\n'.join(
entity_data[i:i + chunk_size])
# Now push the entity data to firecloud
r = fapi.upload_entities(project, workspace, this_data, model)
fapi._check_response_code(r, 200)
return 0
def delete(self):
"""Delete the workspace from FireCloud.
Note:
This action cannot be undone. Be careful!
"""
r = fapi.delete_workspace(self.namespace, self.name)
fapi._check_response_code(r, 202)
def entities(self):
"""List all entities in workspace."""
r = fapi.get_entities_with_type(self.namespace,
self.name, self.api_url)
fapi._check_response_code(r, 200)
edicts = r.json()
return [Entity(e['entityType'], e['name'], e['attributes'])
for e in edicts]
def import_tsv(self, tsv_file):
"""Upload entity data to workspace from tsv loadfile.
Args:
tsv_file (file): Tab-delimited file of entity data
"""
r = fapi.upload_entities_tsv(self.namespace, self.name,
self.tsv_file, self.api_url)
fapi._check_response_code(r, 201)
def get_workspaces(namespace):
# This is based on the fiss list_spaces function but only workspaces are returned
workspaces = api.list_workspaces()
api._check_response_code(workspaces, 200)
for space in workspaces.json():
if namespace == space['workspace']['namespace']:
yield space['workspace']['name']
def validate_monitor_tasks(dependencies, args):
""" Validate that all entries in the supervisor are valid task configurations and
that all permissions requirements are satisfied.
"""
# Make a list of all task configurations needed to supervise
sup_configs = sorted(dependencies.keys())
try:
logging.info("Validating supervisor data...")
# Make an api call to list task configurations in the workspace
r = fapi.list_workspace_configs(args['project'], args['workspace'])
fapi._check_response_code(r, 200)
space_configs = r.json()
# Make a dict for easy lookup later
space_configs = {c["name"]: c for c in space_configs}
# Also make an api call to list methods you have view permissions for
r = fapi.list_repository_methods()
fapi._check_response_code(r, 200)
repo_methods = r.json()
## Put in a form that is more easily searchable: namespace/name:snapshot
repo_methods = {
m['namespace'] + '/' + m['name'] + ':' + str(m['snapshotId'])
for m in repo_methods if m['entityType'] == 'Workflow'
}
valid = True
for config in sup_configs:
# ensure config exists in the workspace
if config not in space_configs:
logging.error("No task configuration for " + config +
" found in " + args['project'] + "/" +
args['workspace'])
valid = False
else:
# Check access permissions for the referenced method
m = space_configs[config]['methodRepoMethod']
ref_method = m['methodNamespace'] + "/" + m[
'methodName'] + ":" + str(m['methodVersion'])
if ref_method not in repo_methods:
logging.error(
config +
" -- You don't have permisson to run the referenced method: "
+ ref_method)
valid = False
except Exception as e:
logging.error("Exception occurred while validating supervisor: " +
str(e))
raise
return False
return valid
def lock(self):
"""Lock this Workspace.
This causes the workspace to behave in a read-only way,
regardless of access permissions.
"""
r = fapi.lock_workspace(self.namespace, self.name, self.api_url)
fapi._check_response_code(r, 204)
self.data['workspace']['isLocked'] = True
return self
def import_entities(self, entities):
"""Upload entity objects.
Args:
entities: iterable of firecloud.Entity objects.
"""
edata = Entity.create_payload(entities)
r = fapi.upload_entities(self.namespace, self.name,
edata, self.api_url)
fapi._check_response_code(r, 201)
def delete_entity(self, etype, entity_id):
"""Delete an entity in this workspace.
Args:
etype (str): Entity type
entity_id (str): Entity name/unique id
"""
r = fapi.delete_entity(self.namespace, self.name, etype,
entity_id, self.api_url)
fapi._check_response_code(r, 202)
def refresh(self):
"""Reload workspace metadata from firecloud.
Workspace metadata is cached in the data attribute of a Workspace,
and may become stale, requiring a refresh().
"""
r = fapi.get_workspace(self.namespace, self.name, self.api_url)
fapi._check_response_code(r, 200)
self.data = r.json()
return self
def test_space_search(self):
# First retrieve information about the space
r = fapi.get_workspace(self.project, self.workspace)
fapi._check_response_code(r, 200)
metadata = r.json()["workspace"]
# Now use part of that info (bucket id) to find the space (name)
result = call_func("space_search", "-b", metadata['bucketName'])
self.assertIn(metadata['name'], ''.join(result))
# Now search for goofy thing that should never be found
self.assertEqual([], call_func("space_search", "-b", '__NoTTHeRe__'))
def remove_attribute(self, attr):
"""Remove attribute from a workspace.
Args:
attr (str): attribute name
"""
update = [fapi._attr_rem(attr)]
r = fapi.update_workspace_attributes(self.namespace, self.name,
update, self.api_url)
self.data["workspace"]["attributes"].pop(attr, None)
fapi._check_response_code(r, 200)
def new(wnamespace,
workspace,
cnamespace,
config,
entity_id,
etype,
expression,
api_url=fapi.PROD_API_ROOT):
r, c = fapi.create_submission(wnamespace, workspace, cnamespace,
config, entity_id, expression, api_url)
fapi._check_response_code(r, 201)
def clone(self, to_namespace, to_name):
"""Clone this workspace.
Args:
to_namespace (str): Target workspace namespace
to_name (str): Target workspace name
"""
r = fapi.clone_workspace(self.namespace, self.name,
to_namespace, to_name, self.api_url)
fapi._check_response_code(r, 201)
return Workspace(to_namespace, to_name, self.api_url)
def call_api(url):
""" Use the firecloud api module to query firecloud
Allows for additional options not included in fiss api"""
result = api.__get(url)
try:
api._check_response_code(result, 200)
result = result.json()
except errors.FireCloudServerError as e:
print("Error with api query ")
print(e)
result = {}
return result
def set_acl(self, role, users):
"""Set access permissions for this workspace
Args:
role (str): Access level
one of {one of "OWNER", "READER", "WRITER", "NO ACCESS"}
users (list(str)): List of users to give role to
"""
acl_updates = [{"email": user, "accessLevel": role} for user in users]
r = fapi.update_workspace_acl(self.namespace, self.name,
acl_updates, self.api_url)
fapi._check_response_code(r, 200)
def get_entity(self, etype, entity_id):
"""Return entity in this workspace.
Args:
etype (str): Entity type
entity_id (str): Entity name/unique id
"""
r = fapi.get_entity(self.namespace, self.name, etype,
entity_id, self.api_url)
fapi._check_response_code(r, 200)
dresp = r.json()
return Entity(etype, entity_id, dresp['attributes'])
def __init__(self,
namespace,
workspace,
submission_id,
api_url=fapi.PROD_API_ROOT):
r = fapi.get_submission(namespace, workspace, submission_id, api_url)
fapi._check_response_code(r, 200)
self.namespace = namespace
self.workspace = workspace
self.submission_id = submission_id
self.api_url = api_url
def __init__(self, namespace, name,
snapshot_id, api_url=fapi.PROD_API_ROOT):
r = fapi.get_method(namespace, name, snapshot_id, api_url)
fapi._check_response_code(r, 200)
data = r.json()
self.namespace = namespace
self.name = name
self.snapshot_id = int(data["snapshotId"])
self.wdl = data["payload"]
self.synopsis = data["synopsis"]
self.documentation = data["documentation"]
self.api_url = api_url
def update_metadata(namespace, workspace, sample_set, data):
fpath = 'gs://{0}/cohort_lists/{1}'
bucket_name = get_bucket_name(namespace, workspace)
for attr, fname in data:
list_path = fpath.format(bucket_name, fname)
update = fapi._attr_set(attr, list_path)
try:
r = fapi.update_entity(namespace, workspace, 'sample_set',
sample_set, [update])
fapi._check_response_code(r, 200)
except ferrors.FireCloudServerError:
pass
def delete_entity_type(args, entity_type, entities):
num_chunks = len(entities) / ENTITY_CHUNK_SIZE
if len(entities) % ENTITY_CHUNK_SIZE != 0:
num_chunks += 1
for i in xrange(0, len(entities), ENTITY_CHUNK_SIZE):
chunk = entities[i:i + ENTITY_CHUNK_SIZE]
print('Deleting chunk %s of %s for type: %s' %
(i / ENTITY_CHUNK_SIZE + 1, num_chunks, entity_type))
resp = fapi.delete_entity_type(args.workspace_namespace,
args.workspace_name, entity_type, chunk)
fapi._check_response_code(resp, 204)
print('Succesfully deleted entities of type: %s' % entity_type)
def copy_entities(self, from_namespace, from_workspace, etype, enames):
"""Copy entities from another workspace.
Args:
from_namespace (str): Source workspace namespace
from_workspace (str): Source workspace name
etype (str): Entity type
enames (list(str)): List of entity names to copy
"""
r = fapi.copy_entities(from_namespace, from_workspace,
self.namespace, self.name, etype, enames,
self.api_url)
fapi._check_response_code(r, 201)
def new(namespace, name, protected=False,
attributes=dict(), api_url=fapi.PROD_API_ROOT):
"""Create a new FireCloud workspace.
Returns:
Workspace: A new FireCloud workspace
Raises:
FireCloudServerError: API call failed.
"""
r = fapi.create_workspace(namespace, name, protected, attributes, api_url)
fapi._check_response_code(r, 201)
return Workspace(namespace, name, api_url)
def update_metadata(namespace, workspace, vcf_lists):
fpath = 'gs://{0}/other_vcf_lists/{1}'
bucket_name = get_bucket_name(namespace, workspace)
for batch, pesr_vcflist, depth_vcflist in vcf_lists:
list_path = fpath.format(bucket_name, pesr_vcflist)
update = fapi._attr_set('other_batches_filtered_pesr_vcf_list', list_path)
r = fapi.update_entity(namespace, workspace, 'sample_set', batch, [update])
fapi._check_response_code(r, 200)
list_path = fpath.format(bucket_name, depth_vcflist)
update = fapi._attr_set('other_batches_filtered_depth_vcf_list', list_path)
r = fapi.update_entity(namespace, workspace, 'sample_set', batch, [update])
fapi._check_response_code(r, 200)
def set_acl(self, role, users):
"""Set permissions for this method.
Args:
role (str): Access level
one of {one of "OWNER", "READER", "WRITER", "NO ACCESS"}
users (list(str)): List of users to give role to
"""
acl_updates = [{"user": user, "role": role} for user in users]
r = fapi.update_repository_method_acl(
self.namespace, self.name, self.snapshot_id,
acl_updates, self.api_url
)
fapi._check_response_code(r, 200)
def configs(self):
"""Get method configurations in a workspace."""
raise NotImplementedError
r = fapi.get_configs(self.namespace, self.name, self.api_url)
fapi._check_response_code(r, 200)
cdata = r.json()
configs = []
for c in cdata:
cnamespace = c['namespace']
cname = c['name']
root_etype = c['rootEntityType']
method_namespace = c['methodRepoMethod']['methodNamespace']
method_name = c['methodRepoMethod']['methodName']
method_version = c['methodRepoMethod']['methodVersion']
def update_metadata(namespace, workspace, cohort_batch_id, pesr_vcflist,
depth_vcflist):
fpath = 'gs://{0}/cohort_vcf_lists/{1}'
bucket_name = get_bucket_name(namespace, workspace)
list_path = fpath.format(bucket_name, pesr_vcflist)
update = fapi._attr_set('cohort_filtered_pesr_vcf_list', list_path)
r = fapi.update_entity(namespace, workspace, 'sample_set', cohort_batch_id,
[update])
fapi._check_response_code(r, 200)
list_path = fpath.format(bucket_name, depth_vcflist)
update = fapi._attr_set('cohort_filtered_depth_vcf_list', list_path)
r = fapi.update_entity(namespace, workspace, 'sample_set', cohort_batch_id,
[update])
fapi._check_response_code(r, 200)