def setUpClass(cls): '''Set up FireCloud etc to run tests''' print("\nStarting high-level CLI tests ...\n", file=sys.stderr) fiss_verbosity = os.environ.get("FISS_TEST_VERBOSITY", None) if fiss_verbosity == None: fiss_verbosity = 0 fcconfig = fccore.config_get_all() fcconfig.set_verbosity(fiss_verbosity) cls.project = fcconfig.project if not cls.project: raise ValueError("Your configuration must define a FireCloud project") # Set up a temp workspace for duration of tests. And in case a previous # test failed, we attempt to unlock & delete before creating anew cls.workspace = getuser() + '_FISS_TEST' ret = call_func("space_exists", "-p", cls.project, "-w", cls.workspace) if ret == True and os.environ.get("REUSE_SPACE", None): return print("\tCreating test workspace ...\n", file=sys.stderr) r = fapi.unlock_workspace(cls.project, cls.workspace) r = fapi.delete_workspace(cls.project, cls.workspace) r = fapi.create_workspace(cls.project, cls.workspace) fapi._check_response_code(r, 201)
def __get_entities(self, etype): """Helper to get entities for a given type.""" r = fapi.get_entities(self.namespace, self.name, etype, self.api_url) fapi._check_response_code(r, 200) return [Entity(e['entityType'], e['name'], e['attributes']) for e in r.json()]
def main(): args = parse_args() resp = fapi.get_entities_with_type(args.workspace_namespace, args.workspace_name) fapi._check_response_code(resp, 200) entities_by_type = {} for entity in resp.json(): entity_type = entity['entityType'] if entity_type not in entities_by_type: entities_by_type[entity_type] = [] entities_by_type[entity_type].append(entity['name']) # Entities are sorted by type: participant, participant_set, sample, # sample_set. FireCloud complains if we delete a participant before deleting # associated participant_set/sample/sample_set. for entity_type in FC_ENTITY_TYPES: if entity_type in entities_by_type: entities = entities_by_type[entity_type] delete_entity_type(args, entity_type, entities) del entities_by_type[entity_type] # Delete the remaining entities where order does not matter. for entity_type, entities in entities_by_type.iteritems(): if entity_type not in FC_ENTITY_TYPES: delete_entity_type(args, entity_type, entities)
def batch_load(project, workspace, headerline, entity_data, chunk_size=500, model='flexible'): """ Submit a large number of entity updates in batches of chunk_size """ # Parse the entity type from the first cell, e.g. "entity:sample_id" # First check that the header is valid # if not valid_headerline(headerline, model): # print("Invalid loadfile header:\n" + headerline) # return 1 update_type = "membership" if headerline.startswith( "membership") else "entity" etype = headerline.split('\t')[0].split(':')[1].replace("_id", "") # Split entity_data into chunks total = int(len(entity_data) / chunk_size) + 1 batch = 0 for i in range(0, len(entity_data), chunk_size): batch += 1 print("Updating {0} {1}s {2}-{3}, batch {4}/{5}".format( etype, update_type, i + 1, min(i + chunk_size, len(entity_data)), batch, total)) this_data = headerline + '\n' + '\n'.join( entity_data[i:i + chunk_size]) # Now push the entity data to firecloud r = fapi.upload_entities(project, workspace, this_data, model) fapi._check_response_code(r, 200) return 0
def delete(self): """Delete the workspace from FireCloud. Note: This action cannot be undone. Be careful! """ r = fapi.delete_workspace(self.namespace, self.name) fapi._check_response_code(r, 202)
def entities(self): """List all entities in workspace.""" r = fapi.get_entities_with_type(self.namespace, self.name, self.api_url) fapi._check_response_code(r, 200) edicts = r.json() return [Entity(e['entityType'], e['name'], e['attributes']) for e in edicts]
def import_tsv(self, tsv_file): """Upload entity data to workspace from tsv loadfile. Args: tsv_file (file): Tab-delimited file of entity data """ r = fapi.upload_entities_tsv(self.namespace, self.name, self.tsv_file, self.api_url) fapi._check_response_code(r, 201)
def get_workspaces(namespace): # This is based on the fiss list_spaces function but only workspaces are returned workspaces = api.list_workspaces() api._check_response_code(workspaces, 200) for space in workspaces.json(): if namespace == space['workspace']['namespace']: yield space['workspace']['name']
def validate_monitor_tasks(dependencies, args): """ Validate that all entries in the supervisor are valid task configurations and that all permissions requirements are satisfied. """ # Make a list of all task configurations needed to supervise sup_configs = sorted(dependencies.keys()) try: logging.info("Validating supervisor data...") # Make an api call to list task configurations in the workspace r = fapi.list_workspace_configs(args['project'], args['workspace']) fapi._check_response_code(r, 200) space_configs = r.json() # Make a dict for easy lookup later space_configs = {c["name"]: c for c in space_configs} # Also make an api call to list methods you have view permissions for r = fapi.list_repository_methods() fapi._check_response_code(r, 200) repo_methods = r.json() ## Put in a form that is more easily searchable: namespace/name:snapshot repo_methods = { m['namespace'] + '/' + m['name'] + ':' + str(m['snapshotId']) for m in repo_methods if m['entityType'] == 'Workflow' } valid = True for config in sup_configs: # ensure config exists in the workspace if config not in space_configs: logging.error("No task configuration for " + config + " found in " + args['project'] + "/" + args['workspace']) valid = False else: # Check access permissions for the referenced method m = space_configs[config]['methodRepoMethod'] ref_method = m['methodNamespace'] + "/" + m[ 'methodName'] + ":" + str(m['methodVersion']) if ref_method not in repo_methods: logging.error( config + " -- You don't have permisson to run the referenced method: " + ref_method) valid = False except Exception as e: logging.error("Exception occurred while validating supervisor: " + str(e)) raise return False return valid
def lock(self): """Lock this Workspace. This causes the workspace to behave in a read-only way, regardless of access permissions. """ r = fapi.lock_workspace(self.namespace, self.name, self.api_url) fapi._check_response_code(r, 204) self.data['workspace']['isLocked'] = True return self
def import_entities(self, entities): """Upload entity objects. Args: entities: iterable of firecloud.Entity objects. """ edata = Entity.create_payload(entities) r = fapi.upload_entities(self.namespace, self.name, edata, self.api_url) fapi._check_response_code(r, 201)
def delete_entity(self, etype, entity_id): """Delete an entity in this workspace. Args: etype (str): Entity type entity_id (str): Entity name/unique id """ r = fapi.delete_entity(self.namespace, self.name, etype, entity_id, self.api_url) fapi._check_response_code(r, 202)
def refresh(self): """Reload workspace metadata from firecloud. Workspace metadata is cached in the data attribute of a Workspace, and may become stale, requiring a refresh(). """ r = fapi.get_workspace(self.namespace, self.name, self.api_url) fapi._check_response_code(r, 200) self.data = r.json() return self
def test_space_search(self): # First retrieve information about the space r = fapi.get_workspace(self.project, self.workspace) fapi._check_response_code(r, 200) metadata = r.json()["workspace"] # Now use part of that info (bucket id) to find the space (name) result = call_func("space_search", "-b", metadata['bucketName']) self.assertIn(metadata['name'], ''.join(result)) # Now search for goofy thing that should never be found self.assertEqual([], call_func("space_search", "-b", '__NoTTHeRe__'))
def remove_attribute(self, attr): """Remove attribute from a workspace. Args: attr (str): attribute name """ update = [fapi._attr_rem(attr)] r = fapi.update_workspace_attributes(self.namespace, self.name, update, self.api_url) self.data["workspace"]["attributes"].pop(attr, None) fapi._check_response_code(r, 200)
def new(wnamespace, workspace, cnamespace, config, entity_id, etype, expression, api_url=fapi.PROD_API_ROOT): r, c = fapi.create_submission(wnamespace, workspace, cnamespace, config, entity_id, expression, api_url) fapi._check_response_code(r, 201)
def clone(self, to_namespace, to_name): """Clone this workspace. Args: to_namespace (str): Target workspace namespace to_name (str): Target workspace name """ r = fapi.clone_workspace(self.namespace, self.name, to_namespace, to_name, self.api_url) fapi._check_response_code(r, 201) return Workspace(to_namespace, to_name, self.api_url)
def call_api(url): """ Use the firecloud api module to query firecloud Allows for additional options not included in fiss api""" result = api.__get(url) try: api._check_response_code(result, 200) result = result.json() except errors.FireCloudServerError as e: print("Error with api query ") print(e) result = {} return result
def set_acl(self, role, users): """Set access permissions for this workspace Args: role (str): Access level one of {one of "OWNER", "READER", "WRITER", "NO ACCESS"} users (list(str)): List of users to give role to """ acl_updates = [{"email": user, "accessLevel": role} for user in users] r = fapi.update_workspace_acl(self.namespace, self.name, acl_updates, self.api_url) fapi._check_response_code(r, 200)
def get_entity(self, etype, entity_id): """Return entity in this workspace. Args: etype (str): Entity type entity_id (str): Entity name/unique id """ r = fapi.get_entity(self.namespace, self.name, etype, entity_id, self.api_url) fapi._check_response_code(r, 200) dresp = r.json() return Entity(etype, entity_id, dresp['attributes'])
def __init__(self, namespace, workspace, submission_id, api_url=fapi.PROD_API_ROOT): r = fapi.get_submission(namespace, workspace, submission_id, api_url) fapi._check_response_code(r, 200) self.namespace = namespace self.workspace = workspace self.submission_id = submission_id self.api_url = api_url
def __init__(self, namespace, name, snapshot_id, api_url=fapi.PROD_API_ROOT): r = fapi.get_method(namespace, name, snapshot_id, api_url) fapi._check_response_code(r, 200) data = r.json() self.namespace = namespace self.name = name self.snapshot_id = int(data["snapshotId"]) self.wdl = data["payload"] self.synopsis = data["synopsis"] self.documentation = data["documentation"] self.api_url = api_url
def update_metadata(namespace, workspace, sample_set, data): fpath = 'gs://{0}/cohort_lists/{1}' bucket_name = get_bucket_name(namespace, workspace) for attr, fname in data: list_path = fpath.format(bucket_name, fname) update = fapi._attr_set(attr, list_path) try: r = fapi.update_entity(namespace, workspace, 'sample_set', sample_set, [update]) fapi._check_response_code(r, 200) except ferrors.FireCloudServerError: pass
def delete_entity_type(args, entity_type, entities): num_chunks = len(entities) / ENTITY_CHUNK_SIZE if len(entities) % ENTITY_CHUNK_SIZE != 0: num_chunks += 1 for i in xrange(0, len(entities), ENTITY_CHUNK_SIZE): chunk = entities[i:i + ENTITY_CHUNK_SIZE] print('Deleting chunk %s of %s for type: %s' % (i / ENTITY_CHUNK_SIZE + 1, num_chunks, entity_type)) resp = fapi.delete_entity_type(args.workspace_namespace, args.workspace_name, entity_type, chunk) fapi._check_response_code(resp, 204) print('Succesfully deleted entities of type: %s' % entity_type)
def copy_entities(self, from_namespace, from_workspace, etype, enames): """Copy entities from another workspace. Args: from_namespace (str): Source workspace namespace from_workspace (str): Source workspace name etype (str): Entity type enames (list(str)): List of entity names to copy """ r = fapi.copy_entities(from_namespace, from_workspace, self.namespace, self.name, etype, enames, self.api_url) fapi._check_response_code(r, 201)
def new(namespace, name, protected=False, attributes=dict(), api_url=fapi.PROD_API_ROOT): """Create a new FireCloud workspace. Returns: Workspace: A new FireCloud workspace Raises: FireCloudServerError: API call failed. """ r = fapi.create_workspace(namespace, name, protected, attributes, api_url) fapi._check_response_code(r, 201) return Workspace(namespace, name, api_url)
def update_metadata(namespace, workspace, vcf_lists): fpath = 'gs://{0}/other_vcf_lists/{1}' bucket_name = get_bucket_name(namespace, workspace) for batch, pesr_vcflist, depth_vcflist in vcf_lists: list_path = fpath.format(bucket_name, pesr_vcflist) update = fapi._attr_set('other_batches_filtered_pesr_vcf_list', list_path) r = fapi.update_entity(namespace, workspace, 'sample_set', batch, [update]) fapi._check_response_code(r, 200) list_path = fpath.format(bucket_name, depth_vcflist) update = fapi._attr_set('other_batches_filtered_depth_vcf_list', list_path) r = fapi.update_entity(namespace, workspace, 'sample_set', batch, [update]) fapi._check_response_code(r, 200)
def set_acl(self, role, users): """Set permissions for this method. Args: role (str): Access level one of {one of "OWNER", "READER", "WRITER", "NO ACCESS"} users (list(str)): List of users to give role to """ acl_updates = [{"user": user, "role": role} for user in users] r = fapi.update_repository_method_acl( self.namespace, self.name, self.snapshot_id, acl_updates, self.api_url ) fapi._check_response_code(r, 200)
def configs(self): """Get method configurations in a workspace.""" raise NotImplementedError r = fapi.get_configs(self.namespace, self.name, self.api_url) fapi._check_response_code(r, 200) cdata = r.json() configs = [] for c in cdata: cnamespace = c['namespace'] cname = c['name'] root_etype = c['rootEntityType'] method_namespace = c['methodRepoMethod']['methodNamespace'] method_name = c['methodRepoMethod']['methodName'] method_version = c['methodRepoMethod']['methodVersion']
def update_metadata(namespace, workspace, cohort_batch_id, pesr_vcflist, depth_vcflist): fpath = 'gs://{0}/cohort_vcf_lists/{1}' bucket_name = get_bucket_name(namespace, workspace) list_path = fpath.format(bucket_name, pesr_vcflist) update = fapi._attr_set('cohort_filtered_pesr_vcf_list', list_path) r = fapi.update_entity(namespace, workspace, 'sample_set', cohort_batch_id, [update]) fapi._check_response_code(r, 200) list_path = fpath.format(bucket_name, depth_vcflist) update = fapi._attr_set('cohort_filtered_depth_vcf_list', list_path) r = fapi.update_entity(namespace, workspace, 'sample_set', cohort_batch_id, [update]) fapi._check_response_code(r, 200)