def test_clone_workspace(self):
"""Test clone_workspace()."""
temp_space = getuser() + '_FISS_TEST_CLONE'
r = fapi.unlock_workspace(self.project, temp_space)
r = fapi.delete_workspace(self.project, temp_space)
r = fapi.clone_workspace(self.project, self.workspace, self.project,
temp_space)
print(r.status_code, r.content)
self.assertEqual(r.status_code, 201)
# Compare new workspace and old workspace
# Cleanup, Delete workspace
r = fapi.delete_workspace(self.project, temp_space)
print(r.status_code, r.content)
self.assertIn(r.status_code, [200, 202])
def setUpClass(cls):
'''Set up FireCloud etc to run tests'''
print("\nStarting high-level CLI tests ...\n", file=sys.stderr)
fiss_verbosity = os.environ.get("FISS_TEST_VERBOSITY", None)
if fiss_verbosity == None:
fiss_verbosity = 0
fcconfig = fccore.config_get_all()
fcconfig.set_verbosity(fiss_verbosity)
cls.project = fcconfig.project
if not cls.project:
raise ValueError("Your configuration must define a FireCloud project")
# Set up a temp workspace for duration of tests. And in case a previous
# test failed, we attempt to unlock & delete before creating anew
cls.workspace = getuser() + '_FISS_TEST'
ret = call_func("space_exists", "-p", cls.project, "-w", cls.workspace)
if ret == True and os.environ.get("REUSE_SPACE", None):
return
print("\tCreating test workspace ...\n", file=sys.stderr)
r = fapi.unlock_workspace(cls.project, cls.workspace)
r = fapi.delete_workspace(cls.project, cls.workspace)
r = fapi.create_workspace(cls.project, cls.workspace)
fapi._check_response_code(r, 201)
def test_space_lock_unlock(self):
args = ("-p", self.project, "-w", self.workspace)
self.assertEqual(0, call_func('space_lock', *args))
# Verify LOCKED worked, by trying to delete
r = fapi.delete_workspace(self.project, self.workspace)
fapi._check_response_code(r, 403)
self.assertEqual(0, call_func('space_unlock', *args))
# Verify UNLOCKED, again by trying to delete
r = fapi.delete_workspace(self.project, self.workspace)
fapi._check_response_code(r, 200)
# Lastly, recreate space in case this test was run in series with others
r = fapi.create_workspace(self.project, self.workspace)
fapi._check_response_code(r, 201)
def delete(self):
"""Delete the workspace from FireCloud.
Note:
This action cannot be undone. Be careful!
"""
r = fapi.delete_workspace(self.namespace, self.name)
fapi._check_response_code(r, 202)
def delete_workspace(workspace, project, retry=0):
# don't retry too much
if retry > 3:
print(
f"WARNING: internal errors not resolved by retries. Workspace {project}/{workspace} was NOT deleted."
)
print("Exiting.")
exit(1)
sleep_time = 5 # seconds to wait between retries
# try to delete the workspace
try:
response = fapi.delete_workspace(project, workspace)
status_code = response.status_code
if status_code == 202:
print(f"Workspace {project}/{workspace} was successfully deleted!")
return 1
elif status_code == 403:
print(
f"WARNING: Insufficient permissions to delete {project}/{workspace}."
)
return 0
elif status_code == 404:
print(f"Workspace {project}/{workspace} not found. Cool!")
return 1
elif status_code == 500:
# try again
incremented_retry = retry + 1
print(
f"Retrying deletion of {project}/{workspace} (attempt {incremented_retry}) after {sleep_time} seconds"
)
sleep(sleep_time)
return delete_workspace(workspace, project, incremented_retry)
else:
print(f"Unknown status code: {status_code}. Exiting.")
exit(1)
except Exception:
# try again
incremented_retry = retry + 1
print(
f"Retrying deletion of {project}/{workspace} (attempt {incremented_retry}) after {sleep_time} seconds"
)
sleep(sleep_time)
return delete_workspace(workspace, project, incremented_retry)
def setUpClass(cls, msg=""):
'''Set up FireCloud etc to run tests'''
print("\nStarting low-level api tests ...\n", file=sys.stderr)
fiss_verbosity = os.environ.get("FISS_TEST_VERBOSITY", None)
if fiss_verbosity == None:
fiss_verbosity = 0
fcconfig = fccore.config_parse()
cls.project = fcconfig.project
if not cls.project:
raise ValueError("Your configuration must define a FireCloud project")
fcconfig.set_verbosity(fiss_verbosity)
# Set up a temp workspace for duration of tests; and in case a previous
# test failed, attempt to unlock & delete before creating anew. Note
# that bc we execute space create/delete here, their tests are NO-OPs
cls.workspace = getuser() + '_FISS_TEST'
r = fapi.unlock_workspace(cls.project, cls.workspace)
r = fapi.delete_workspace(cls.project, cls.workspace)
r = fapi.create_workspace(cls.project, cls.workspace)
fapi._check_response_code(r, 201)
def tearDownClass(cls):
print("\nFinishing high-level CLI tests ...\n", file=sys.stderr)
r = fapi.delete_workspace(cls.project, cls.workspace)
def main():
if len(sys.argv) < 2:
return
global billing_project
global template_workspace_name
parser = argparse.ArgumentParser(prog="python " + sys.argv[0],
add_help=False)
subparser = parser.add_subparsers(dest="cmd")
delete_workspace = subparser.add_parser('delete_workspace',
help='delete workspace')
delete_workspace.add_argument('--workspace-name',
dest="workspace_name",
help="name of the workspace")
clone_workspace = subparser.add_parser(
'clone_workspace', help='clone from existing workspace')
clone_workspace.add_argument('--source-work-space',
dest='src_work_space',
help="name of source workspace")
clone_workspace.add_argument('--destination-work-space',
dest='dest_work_space',
help="name of destination workspace")
get_data_info = subparser.add_parser('get_participant_table',
help='get participant.tsv')
get_data_info.add_argument('--workspace-name',
dest="workspace_name",
help="name of the workspace")
get_data_info.add_argument('--participant-table-name',
dest="participant_table_name",
help="name of sample table")
get_data_info.add_argument('--output-name',
dest="output_table_name",
required=False,
default="participant.tsv",
help="name of output tsv")
create_participant_lane = subparser.add_parser(
'create_participant_lane',
help='create participant_lane/lane_set_id tables')
create_participant_lane.add_argument('--input-name',
dest="input_participant_table_name",
required=False,
default="participant.tsv",
help="input participant table name")
create_participant_lane.add_argument(
'--output-prefix',
dest="output_prefix",
required=False,
help="name of output prefix for the lanes")
upload_participant_lane = subparser.add_parser(
'upload_participant',
help=
'uploads the participant_lane_set, _lane_membership and _lane_entity files'
)
upload_participant_lane.add_argument('--workspace-name',
dest="workspace_name",
help="name of the workspace")
upload_participant_lane.add_argument('--input-prefix',
dest="input_prefix",
help="name of the input prefix")
upload_workflow = subparser.add_parser(
'upload_workflow', help='uploads wdl to --workspace-name')
upload_workflow.add_argument('--workspace-name',
dest="workspace_name",
help="name of the workspace")
upload_workflow.add_argument('--method',
dest="method",
help="name of the input prefix")
upload_workflow.add_argument('--wdl',
dest="wdl",
help="name of the input prefix")
upload_config = subparser.add_parser('upload_config',
help='upload config information')
upload_config.add_argument('--workspace-name',
dest="workspace_name",
help="name of the workspace")
upload_config.add_argument('--chemistry',
dest="chemistry",
choices=["V2", "V3"],
help="chemistry")
upload_config.add_argument(
'--counting-mode',
dest="counting_mode",
choices=["sc_rna", "sn_rna"],
help="counting mode: whether to count intronic alignments")
upload_config.add_argument('--species',
dest="species",
choices=["human", "mouse"],
help="species")
submit_workflow = subparser.add_parser('submit_workflow',
help='submit a workflow run')
submit_workflow.add_argument('--workspace-name',
dest="workspace_name",
help="name of the workspace")
submit_workflow.add_argument('--workflow-repo',
dest="workflow_repo",
help="workflow repo name")
submit_workflow.add_argument('--workflow-name',
dest="workflow_name",
help="workflow name")
submit_workflow.add_argument('--entity-id',
dest="entity_id",
help="entity id")
get_status = subparser.add_parser('get_status',
help='get status of a submission')
get_status.add_argument('--workspace-name',
dest="workspace_name",
help="name of the workspace")
get_status.add_argument('--submission-id',
dest="submission_id",
help="submission_id")
# show help when no arguments supplied
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
args = parser.parse_args()
# new_workspace_name = "DCP2_Optimus_template_KMK_v1"
if args.cmd == 'delete_workspace':
print("Delete existing workspace ", args.workspace_name)
delete_status = fapi.delete_workspace(billing_project,
args.workspace_name)
elif args.cmd == 'clone_workspace':
print("Cloning a new workspace from template", args.src_work_space)
status = create_newworkspace(billing_project, args.src_work_space,
args.dest_work_space)
elif args.cmd == 'get_participant_table':
print("Get information from workspace", args.workspace_name)
r = fapi.get_entities_tsv(billing_project, args.workspace_name,
args.participant_table_name)
with open(args.output_table_name, 'w') as fout:
fout.write(r.content.decode())
elif args.cmd == 'create_participant_lane':
parse_terra.create_output_files(args.input_participant_table_name,
args.output_prefix)
elif args.cmd == 'upload_participant':
upload_tables(args.input_prefix + ".tsv", billing_project,
args.workspace_name)
upload_tables(args.input_prefix + "_membership.tsv", billing_project,
args.workspace_name)
upload_tables(args.input_prefix + "_entity.tsv", billing_project,
args.workspace_name)
elif args.cmd == 'upload_workflow':
r = fapi.update_repository_method(args.workspace_name, args.method,
"args.synopsis", args.wdl,
"comment.txt", "args.comment")
with open("response.txt", 'w') as fout:
fout.write(r.content.decode())
elif args.cmd == 'upload_config':
work_space_config = fapi.get_workspace_config(billing_project,
args.workspace_name,
args.workspace_name,
"Optimus")
work_space_json = work_space_config.json()
work_space_json['inputs'] = json_templates.optimus_inputs
work_space_json['outputs'] = json_templates.optimus_outputs
if args.chemistry == "V2":
work_space_json['inputs']['Optimus.chemistry'] = '\"tenX_v2\"'
work_space_json['inputs'][
'Optimus.whitelist'] = 'workspace.whitelist_v2'
if args.chemistry == "V3":
work_space_json['inputs']['Optimus.chemistry'] = '\"tenX_v3\"'
work_space_json['inputs'][
'Optimus.whitelist'] = 'workspace.whitelist_v3'
if args.chemistry == "sn_rna":
work_space_json['inputs']['Optimus.counting_mode'] = "\"sn_rna\""
if args.chemistry == "sc_rna":
work_space_json['inputs']['Optimus.counting_mode'] = "\"sc_rna\""
if args.species == "human":
work_space_json['inputs'][
'Optimus.annotations_gtf'] = 'workspace.human_annotations_gtf'
work_space_json['inputs'][
'Optimus.ref_genome_fasta'] = 'workspace.human_ref_genome_fasta'
work_space_json['inputs'][
'Optimus.tar_star_reference'] = 'workspace.human_tar_star_reference'
if args.species == "mouse":
work_space_json['inputs'][
'Optimus.annotations_gtf'] = 'workspace.mouse_annotations_gtf'
work_space_json['inputs'][
'Optimus.ref_genome_fasta'] = 'workspace.mouse_ref_genome_fasta'
work_space_json['inputs'][
'Optimus.tar_star_reference'] = 'workspace.mouse_tar_star_reference'
updated_workflow = fapi.update_workspace_config(
billing_project, args.workspace_name, args.workspace_name,
"Optimus", work_space_json)
if updated_workflow.status_code != 200:
print("ERROR :" + updated_workflow.content)
sys.exit(1)
else:
print("updated successfully")
elif args.cmd == 'submit_workflow':
# Launching the Updated Monitor Submission Workflow
response = fapi.get_entities_with_type(billing_project,
args.workspace_name)
entities = response.json()
for ent in entities:
ent_name = ent['name']
ent_type = ent['entityType']
ent_attrs = ent['attributes']
submisson_response = fapi.create_submission(
billing_project,
args.workspace_name,
args.workflow_repo,
args.workflow_name,
entity=args.entity_id,
etype="participant_lane_set",
expression=None,
use_callcache=True)
if submisson_response.status_code != 201:
print(submisson_response.content)
sys.exit(1)
else:
print("Successfully Created Submisson")
with open('response.txt', 'w') as fout:
# json.dump(submisson_response.json(), fout)
fout.write(submisson_response.json()['submissionId'] + '\n')
# r = create_workspace_config("broadgdac", args.workspace_name, body):
# print(r.content.decode())
elif args.cmd == 'get_status':
res = fapi.get_submission(billing_project, args.workspace_name,
args.submission_id)
print(res.content.decode())
def tearDownClass(cls):
print("\nFinishing high-level CLI tests ...\n", file=sys.stderr)
if not os.environ.get("REUSE_SPACE", None):
fapi.delete_workspace(cls.project, cls.workspace)
