def setUpClass(cls):
'''Set up FireCloud etc to run tests'''
print("\nStarting high-level CLI tests ...\n", file=sys.stderr)
fiss_verbosity = os.environ.get("FISS_TEST_VERBOSITY", None)
if fiss_verbosity == None:
fiss_verbosity = 0
fcconfig = fccore.config_get_all()
fcconfig.set_verbosity(fiss_verbosity)
cls.project = fcconfig.project
if not cls.project:
raise ValueError("Your configuration must define a FireCloud project")
# Set up a temp workspace for duration of tests. And in case a previous
# test failed, we attempt to unlock & delete before creating anew
cls.workspace = getuser() + '_FISS_TEST'
ret = call_func("space_exists", "-p", cls.project, "-w", cls.workspace)
if ret == True and os.environ.get("REUSE_SPACE", None):
return
print("\tCreating test workspace ...\n", file=sys.stderr)
r = fapi.unlock_workspace(cls.project, cls.workspace)
r = fapi.delete_workspace(cls.project, cls.workspace)
r = fapi.create_workspace(cls.project, cls.workspace)
fapi._check_response_code(r, 201)
def get_or_create_workspace(workspace_namespace, workspace_name):
ws = fapi.get_workspace(workspace_namespace, workspace_name)
if ws.status_code == 404:
ws = fapi.create_workspace(workspace_namespace, workspace_name)
if ws.status_code != 201:
raise ValueError('Unable to create workspace')
return ws.json()
else:
return ws.json()['workspace']
def new(namespace, name, protected=False,
attributes=dict(), api_url=fapi.PROD_API_ROOT):
"""Create a new FireCloud workspace.
Returns:
Workspace: A new FireCloud workspace
Raises:
FireCloudServerError: API call failed.
"""
r = fapi.create_workspace(namespace, name, protected, attributes, api_url)
fapi._check_response_code(r, 201)
return Workspace(namespace, name, api_url)
def test_space_lock_unlock(self):
args = ("-p", self.project, "-w", self.workspace)
self.assertEqual(0, call_func('space_lock', *args))
# Verify LOCKED worked, by trying to delete
r = fapi.delete_workspace(self.project, self.workspace)
fapi._check_response_code(r, 403)
self.assertEqual(0, call_func('space_unlock', *args))
# Verify UNLOCKED, again by trying to delete
r = fapi.delete_workspace(self.project, self.workspace)
fapi._check_response_code(r, 200)
# Lastly, recreate space in case this test was run in series with others
r = fapi.create_workspace(self.project, self.workspace)
fapi._check_response_code(r, 201)
def setUpClass(cls, msg=""):
'''Set up FireCloud etc to run tests'''
print("\nStarting low-level api tests ...\n", file=sys.stderr)
fiss_verbosity = os.environ.get("FISS_TEST_VERBOSITY", None)
if fiss_verbosity == None:
fiss_verbosity = 0
fcconfig = fccore.config_parse()
cls.project = fcconfig.project
if not cls.project:
raise ValueError("Your configuration must define a FireCloud project")
fcconfig.set_verbosity(fiss_verbosity)
# Set up a temp workspace for duration of tests; and in case a previous
# test failed, attempt to unlock & delete before creating anew. Note
# that bc we execute space create/delete here, their tests are NO-OPs
cls.workspace = getuser() + '_FISS_TEST'
r = fapi.unlock_workspace(cls.project, cls.workspace)
r = fapi.delete_workspace(cls.project, cls.workspace)
r = fapi.create_workspace(cls.project, cls.workspace)
fapi._check_response_code(r, 201)
def main():
parser = argparse.ArgumentParser(
description=
'create FireCloud workspace for a specific project + cohort + access type'
)
parser.add_argument("project_name",
help="the name of the project. e.g: TCGA")
parser.add_argument("cohort_name", help="name_of_cancer_cohort. e.g: LUAD")
parser.add_argument(
"billing_project",
help=
"name of billing project to create the workspace under. e.g: broad-firecloud-tcga"
)
parser.add_argument(
"ws_suffix",
help=
"descriptive suffix to add to the workspace auto-generated name. e.g: ControlledAccess_hg38_V1-0_DATA"
)
parser.add_argument(
"-a",
"--auth_domain",
help=
"authorization domain. for dbGaP controlled access the domain name is TCGA-dbGaP-Authorized.",
default="")
args = parser.parse_args()
#STEP 1:
#Create new directory for the cohort and switch wd to this directory
if args.auth_domain:
new_dir_name = args.project_name + "-" + args.cohort_name + "_" + args.auth_domain
else:
new_dir_name = args.project_name + "-" + args.cohort_name
os.mkdir(new_dir_name)
print("Created new directory for the {0} cohort".format(args.cohort_name))
os.chdir(new_dir_name)
print("Switched working directory to ./{0}".format(new_dir_name))
#STEP 2:
#Create creteria for downloading manifest and then download it.
#Right now the file types that are selected for a new workspace depend on whether that workspace is to have open/controlled access to the GDC data portal.
#This code will need to be redesigned, or new keys will have to be added to the dictionary if this assumption ever changes.
if args.auth_domain:
file_types = FILE_TYPE_DICT[args.auth_domain]
else:
file_types = FILE_TYPE_DICT["default"]
filters = dict()
filters["cases.project.program.name"] = [args.project_name]
filters["cases.project.project_id"] = [
args.project_name + "-" + args.cohort_name
]
filters["files.access"] = file_types
#Following directions from the GDC, we were told that controlled access workspaces should not contain BAM files
if args.auth_domain:
filters["files.data_format"] = [
"BCR XML", "TXT", "VCF", "TSV", "MAF", "XLSX"
]
else:
filters["files.data_format"] = [
"BCR XML", "TXT", "VCF", "TSV", "MAF", "XLSX"
]
filt_json = build_filter_json(filters)
#Download manifest file to the new directory
manifest_filename = download_manifest(filt_json)
print("manifest downloaded")
#Step 3:
#Run fcgdctools on the manifest file
if args.project_name == "TARGET":
fcgdctools_command = "genFcWsLoadFiles -c " + manifest_filename + ">genFcWsLoadFiles_output.txt"
else:
fcgdctools_command = "genFcWsLoadFiles " + manifest_filename + ">genFcWsLoadFiles_output.txt"
print(
"Executing command {0}\nPlease check the output file to see progress and check for errors."
.format(fcgdctools_command))
os.system(fcgdctools_command)
#Step 4:
#Prepare attributes to be loaded
workspace_attribute_filename = manifest_filename.split(
".")[0] + "_workspace_attributes.txt"
attribute_list = prepare_workspace_attribute_list(
workspace_attribute_filename, args.auth_domain)
#Step 5:
#Create the new workspace on FireCloud
workspace_name = "{0}_{1}_{2}".format(args.project_name, args.cohort_name,
args.ws_suffix)
print("New workspace name is: {0}\nPreparing to create workspace.".format(
workspace_name))
api.create_workspace(args.billing_project, workspace_name,
args.auth_domain, attribute_list)
#Step 6:
#Upload data model .tsv files to the newly created workspace
data_model_file_prefix = manifest_filename.split(".")[0]
data_files = [
"participants", "participant_sets_membership", "samples",
"sample_sets_membership", "pairs", "pair_sets_membership"
]
for filetype in data_files:
full_name = data_model_file_prefix + "_" + filetype + ".txt"
if os.path.exists(full_name):
print("Uploading file {0}".format(full_name))
api.upload_entities_tsv(args.billing_project, workspace_name,
full_name)
#Step 7:
#Create and Upload method configurations for downloading files to the new workspace
downloadable_attrs = list_downloadable_attrs(
data_model_file_prefix, ["participant", "sample", "pair"])
print("The downloadable attributes are:")
for attr in downloadable_attrs:
print(attr[0])
create_method_configs(args.billing_project, workspace_name,
downloadable_attrs, args.auth_domain)
