def set_up_auth(self, mock_header):
# set up authentication options for the tests
temp_dir = tempfile.mkdtemp()
secrets_file = temp_dir + 'fake_secrets.json'
service_account_key = os.path.join(temp_dir, 'fake_key.json')
username = "******"
password = "******"
url = "https://fake_url"
auth = {"url": url, "username": username, "password": password}
with open(secrets_file, 'w') as f:
json.dump(auth, f)
mock_header.return_value = CromwellAuth(
url=url, header={"Authorization": "bearer fake_token"}, auth=None)
auth_options = (
CromwellAuth.harmonize_credentials(**auth), # HTTPBasicAuth
CromwellAuth.harmonize_credentials(
**{"secrets_file": secrets_file}), # Secret file
CromwellAuth.harmonize_credentials(**{
"service_account_key": service_account_key,
"url": url
}), # OAuth
CromwellAuth.harmonize_credentials(url=url), # No Auth
)
return auth_options
def test_harmonize_credentials_only_takes_one_auth_type(mock_header):
url = 'https://cromwell.server.org'
expected_auth = CromwellAuth(url=url,
header={"Authorization": "bearer fake_token"},
auth=None)
mock_header.return_value = expected_auth
with pytest.raises(ValueError):
CromwellAuth.harmonize_credentials(**auth_types)
def test_harmonize_credentials_from_service_account_key(mock_header):
service_account_key = 'fake_key.json'
url = 'https://cromwell.server.org'
expected_auth = CromwellAuth(url=url,
header={"Authorization": "bearer fake_token"},
auth=None)
mock_header.return_value = expected_auth
auth = CromwellAuth.harmonize_credentials(
url=url, service_account_key=service_account_key)
assert auth == expected_auth
def get_cromwell_auth(settings):
cromwell_url = settings.get("cromwell_url")
if settings.get("use_caas"):
return CromwellAuth.harmonize_credentials(
url=cromwell_url, service_account_key=settings.get("caas_key"))
return CromwellAuth.harmonize_credentials(
url=cromwell_url,
username=settings.get("cromwell_user"),
password=settings.get("cromwell_password"),
)
def test_harmonize_credentials_from_secrets_file():
username = "******"
password = "******"
url = "https://fake_url"
expected_auth = CromwellAuth(url=url,
header=None,
auth=requests.auth.HTTPBasicAuth(
username, password))
auth = CromwellAuth.harmonize_credentials(
secrets_file=auth_types['secrets_file']['secrets_file'])
assert auth.auth == expected_auth.auth
assert auth.header == expected_auth.header
def test_harmonize_credentials_from_service_account_key_content(mock_header):
service_account_key = {
'client_email': 'fake_email',
'token_uri': 'fake_uri'
}
url = 'https://cromwell.server.org'
expected_auth = CromwellAuth(url=url,
header={"Authorization": "bearer fake_token"},
auth=None)
mock_header.return_value = expected_auth
auth = CromwellAuth.harmonize_credentials(
url=url, service_account_key=service_account_key)
assert auth == expected_auth
def test_harmonize_credentials_user_password():
username = '******'
password = '******'
url = 'https://cromwell.server.org'
expected_auth = CromwellAuth(url=url,
header=None,
auth=requests.auth.HTTPBasicAuth(
username, password))
auth = CromwellAuth.harmonize_credentials(username=username,
password=password,
url=url)
assert auth.auth == expected_auth.auth
assert auth.header == expected_auth.header
def test_if_compose_oauth_options_for_jes_backend_cromwell_can_deal_with_null_workflow_options(
self, ):
test_url = 'https://fake_url'
test_service_account_key = 'data/fake_account_key.json'
with open(test_service_account_key, 'r') as f:
test_service_account_key_content = json.load(f)
test_auth = CromwellAuth(
url=test_url,
header={"Authorization": "bearer fake_token"},
auth=None,
service_key_content=test_service_account_key_content,
)
result_options = utils.compose_oauth_options_for_jes_backend_cromwell(
test_auth)
result_options_in_dict = json.loads(result_options.getvalue())
assert (result_options_in_dict['google_project'] ==
test_service_account_key_content['project_id'])
assert (result_options_in_dict['google_compute_service_account'] ==
test_service_account_key_content['client_email'])
assert result_options_in_dict[
'user_service_account_json'] == json.dumps(
test_service_account_key_content)
def test_compose_oauth_options_for_jes_backend_cromwell_add_required_fields_to_workflow_options(
self, ):
test_url = 'https://fake_url'
test_service_account_key = 'data/fake_account_key.json'
with open(test_service_account_key, 'r') as f:
test_service_account_key_content = json.load(f)
test_auth = CromwellAuth(
url=test_url,
header={"Authorization": "bearer fake_token"},
auth=None,
service_key_content=test_service_account_key_content,
)
result_options = utils.compose_oauth_options_for_jes_backend_cromwell(
test_auth, self.options_file_BytesIO)
# use .decode('utf-8') for Python3.5 compatibility
result_options_in_dict = json.loads(
result_options.getvalue().decode('utf-8'))
assert (
# use .decode('utf-8') for Python3.5 compatibility
result_options_in_dict['read_from_cache'] == json.loads(
self.options_file_BytesIO.getvalue().decode('utf-8'))
['read_from_cache'])
assert (result_options_in_dict['google_project'] ==
test_service_account_key_content['project_id'])
assert (result_options_in_dict['google_compute_service_account'] ==
test_service_account_key_content['client_email'])
assert result_options_in_dict[
'user_service_account_json'] == json.dumps(
test_service_account_key_content)
def submit_to_cromwell(args):
# Get authentication (as no authentication)
auth = CromwellAuth.from_no_authentication(
url="http://localhost:{}".format(args.webservice_port))
response = api.submit(auth=auth,
wdl_file=args.workflow_source,
inputs_files=args.workflow_inputs,
dependencies=args.workflow_dependencies,
validate_labels=True)
# FIXME - need to test this first a bit
print(response)
def test_cli_command_works_with_service_account_auth(mock_header,
service_account_auth):
"""Use the submit command as an example to prove CLI works with u/p auth."""
expected_auth = CromwellAuth(
url="https://fake-cromwell",
header={"Authorization": "bearer fake_token"},
auth=None,
)
mock_header.return_value = expected_auth
user_inputs = [
"submit",
"--wdl-file",
"fake.wdl",
"--inputs-files",
"fake.json",
] + service_account_auth
command, args = cli_parser(user_inputs)
def test_harmonize_credentials_from_no_authentication():
url = "https://fake_url"
expected_auth = CromwellAuth(url=url, header=None, auth=None)
auth = CromwellAuth.harmonize_credentials(url=url)
assert auth.auth == expected_auth.auth
assert auth.header == expected_auth.header
def parser(arguments=None):
# TODO: dynamically walk through the commands and automatcally create parsers here
main_parser = DefaultHelpParser()
# Check the installed version of Cromwell-tools
main_parser.add_argument(
'-V',
'--version',
action='version',
version=f'%(prog)s {cromwell_tools_version}',
)
subparsers = main_parser.add_subparsers(help='sub-command help',
dest='command')
# sub-commands of cromwell-tools
submit = subparsers.add_parser(
'submit',
help='submit help',
description='Submit a WDL workflow on Cromwell.')
wait = subparsers.add_parser(
'wait',
help='wait help',
description='Wait for one or more running workflow to finish.',
)
status = subparsers.add_parser(
'status',
help='status help',
description='Get the status of one or more workflows.',
)
abort = subparsers.add_parser(
'abort',
help='abort help',
description='Request Cromwell to abort a running workflow by UUID.',
)
release_hold = subparsers.add_parser(
'release_hold',
help='release_hold help',
description='Request Cromwell to release the hold on a workflow.',
)
metadata = subparsers.add_parser(
'metadata',
help='metadata help',
description=
'Retrieve the workflow and call-level metadata for a specified workflow by UUID.',
)
query = subparsers.add_parser(
'query',
help='query help',
description='[NOT IMPLEMENTED IN CLI] Query for workflows.',
)
health = subparsers.add_parser(
'health',
help='health help',
description=
'Check that cromwell is running and that provided authentication is valid.',
)
task_runtime = subparsers.add_parser(
'task_runtime',
help='task_runtime help',
description=
'Output tsv breakdown of task runtimes by execution event categories',
)
# cromwell url and authentication arguments apply to all sub-commands
cromwell_sub_commands = (
submit,
wait,
status,
abort,
release_hold,
metadata,
query,
health,
task_runtime,
)
auth_args = {
'url':
'The URL to the Cromwell server. e.g. "https://cromwell.server.org/"',
'username':
'******',
'password':
'******',
'secrets_file':
'Path to the JSON file containing username, password, and url fields.',
'service_account_key':
'Path to the JSON key file for authenticating with CaaS.',
}
def add_auth_args(subcommand_parser):
for arg_dest, help_text in auth_args.items():
subcommand_parser.add_argument(
'--{arg}'.format(arg=arg_dest.replace('_', '-')),
dest=arg_dest,
default=None,
type=str,
help=help_text,
)
# TODO: this should be a group which is called authentication
for p in cromwell_sub_commands:
add_auth_args(p)
# submit arguments
submit.add_argument(
'-w',
'--wdl-file',
dest='wdl_file',
type=str,
required=True,
help='Path to the workflow source file to submit for execution.',
)
submit.add_argument(
'-i',
'--inputs-files',
dest='inputs_files',
nargs='+',
type=str,
required=True,
help=
'Path(s) to the input file(s) containing input data in JSON format, separated by space.',
)
submit.add_argument(
'-d',
'--deps-file',
dest='dependencies',
nargs='+',
type=str,
help=
'Path to the Zip file containing dependencies, or a list of raw dependency files to '
'be zipped together separated by space.',
)
submit.add_argument(
'-o',
'--options-file',
dest='options_file',
type=str,
help='Path to the Cromwell configs JSON file.',
)
# TODO: add a mutually exclusive group to make it easy to add labels for users
submit.add_argument(
'-l',
'--label-file',
dest='label_file',
type=str,
default=None,
help=
'Path to the JSON file containing a collection of key/value pairs for workflow labels.',
)
submit.add_argument(
'-c',
'--collection-name',
dest='collection_name',
type=str,
default=None,
help=
'Collection in SAM that the workflow should belong to, if use CaaS.',
)
submit.add_argument(
'--on-hold',
dest='on_hold',
type=bool,
default=False,
help='Whether to submit the workflow in "On Hold" status.',
)
submit.add_argument(
'--validate-labels',
dest='validate_labels',
type=bool,
default=False,
help='Whether to validate cromwell labels.',
)
# wait arguments
wait.add_argument('workflow_ids', nargs='+')
wait.add_argument(
'--timeout-minutes',
dest='timeout_minutes',
type=int,
default=120,
help='number of minutes to wait before timeout.',
)
wait.add_argument(
'--poll-interval-seconds',
dest='poll_interval_seconds',
type=int,
default=30,
help='seconds between polling cromwell for workflow status.',
)
wait.add_argument(
'--silent',
dest='verbose',
action='store_false',
help=
'whether to silently print verbose workflow information while polling cromwell.',
)
# status arguments
status.add_argument(
'--uuid',
required=True,
help='A Cromwell workflow UUID, which is the workflow identifier.',
)
# abort arguments
abort.add_argument(
'--uuid',
required=True,
help='A Cromwell workflow UUID, which is the workflow identifier.',
)
# release_hold arguments
release_hold.add_argument(
'--uuid',
required=True,
help='A Cromwell workflow UUID, which is the workflow identifier.',
)
# metadata arguments
metadata.add_argument(
'--uuid',
required=True,
help='A Cromwell workflow UUID, which is the workflow identifier.',
)
# TODO: add a mutually exclusive group to make it fail early
metadata.add_argument(
'--includeKey',
nargs='+',
default=None,
help=
'When specified key(s) to include from the metadata. Matches any key starting with the value. May not be used with excludeKey.',
)
metadata.add_argument(
'--excludeKey',
nargs='+',
default=None,
help=
'When specified key(s) to exclude from the metadata. Matches any key starting with the value. May not be used with includeKey.',
)
metadata.add_argument(
'--expandSubWorkflows',
default=False,
help=
'When true, metadata for sub workflows will be fetched and inserted automatically in the metadata response.',
)
either_runtime = task_runtime.add_mutually_exclusive_group(required=True)
either_runtime.add_argument(
'--metadata',
dest='metadata',
help='Metadata json file to calculate cost on',
)
either_runtime.add_argument(
'--uuid',
dest='uuid',
help='A Cromwell workflow UUID, which is the workflow identifier.',
)
# query arguments
# TODO: implement CLI entry for query API.
# group all of the arguments
args = vars(main_parser.parse_args(arguments))
# Return help messages if no arguments provided
if not args['command']:
main_parser.error("No commands/arguments provided!")
# TODO: see if this can be moved or if the commands can be populated from above
if args['command'] in (
'submit',
'wait',
'status',
'abort',
'release_hold',
'health',
'metadata',
'task_runtime',
):
auth_arg_dict = {k: args.get(k) for k in auth_args.keys()}
auth = CromwellAuth.harmonize_credentials(**auth_arg_dict)
args['auth'] = auth
for k in auth_args:
if k in args:
del args[k]
command = getattr(CromwellAPI, args['command'], False)
if not command:
try:
command = diagnostic_index[args['command']]
except KeyError:
raise KeyError(f"{args['command']} is not a valid command.")
del args['command']
return command, args
def run_wdl(ls: Server, params: Tuple[RunWDLParams]):
wdl_uri = params[0].wdl_uri
wdl_path = urlparse(wdl_uri).path
_, wdl = _parse_wdl(ls, wdl_uri)
if not wdl:
return ls.show_message('Unable to submit: WDL contains error(s)', MessageType.Error)
config = _get_client_config(ls)
auth = CromwellAuth.from_no_authentication(config.cromwell.url)
workflow = cromwell_api.submit(
auth, wdl_path, raise_for_status=True,
).json()
id = workflow['id']
title = 'Workflow {} for {}'.format(id, wdl_path)
_progress(ls, 'start', {
'id': id,
'title': title,
'cancellable': True,
'message': workflow['status'],
})
status: str = ''
while True:
if status != workflow['status']:
status = workflow['status']
if status == 'Succeeded':
message_type = MessageType.Info
elif status in ('Aborting', 'Aborted'):
message_type = MessageType.Warning
elif status == 'Failed':
message_type = MessageType.Error
else:
_progress(ls, 'report', {
'id': id,
'message': status,
})
continue
_progress(ls, 'done', {
'id': id,
})
message = '{}: {}'.format(title, status)
ls.show_message(message, message_type)
diagnostics = _parse_failures(wdl, wdl_uri, id, auth)
return ls.publish_diagnostics(wdl_uri, diagnostics)
sleep(config.cromwell.pollSec)
if id in ls.aborting_workflows:
workflow = cromwell_api.abort(
id, auth, raise_for_status=True,
).json()
ls.aborting_workflows.remove(id)
continue
try:
workflow = cromwell_api.status(
id, auth, raise_for_status=True,
).json()
except HTTPError as e:
ls.show_message_log(str(e), MessageType.Error)
def calculate_metric(template_values, scan_values):
merged_values = {
k: v
for (k, v) in (template_values.items() + scan_values.items())
}
merged_json_file, merged_json_path = tempfile.mkstemp()
with open(merged_json_path, 'w') as f:
json.dump(merged_values, f)
cromwell_auth = CromwellAuth(url=args.cromwell_server,
header={'Authorization': 'bearer fake_token'},
auth=None)
with open(args.workflow_wdl, 'r') as w, open(merged_json_path, 'r') as j:
submit = CromwellAPI.submit(cromwell_auth, w, j)
workflow_id = submit.json()['id']
logger.info('Submitted workflow: ' + workflow_id)
time.sleep(5)
logger.info('Waiting for workflow to complete...')
# Query workflow status indefinitely until success or failure returned.
# If success returned, attempt to retrieve objective_value from metadata and return.
# If failure returned or if exception raised during metadata retreival, return bad_value.
try:
while True:
try:
CromwellAPI.wait([workflow_id],
cromwell_auth,
timeout_minutes=600,
poll_interval_seconds=20,
verbose=False)
response = CromwellAPI.status(workflow_id, cromwell_auth)
status = response.json()['status']
if status == 'Succeeded':
logger.info('Workflow succeeded...')
break
except WorkflowFailedException:
logger.info('Workflow failed, returning bad value...')
return bad_value
except Exception as e:
logger.info(e)
logger.info(
'Cromwell exception, retrying wait and status check...')
logger.info('Getting metadata...')
session = retry_session(retries=10)
metadata = session.post(
url=cromwell_auth.url +
CromwellAPI._metadata_endpoint.format(uuid=workflow_id),
auth=cromwell_auth.auth,
headers=cromwell_auth.header)
workflow_name = metadata.json()['workflowName']
objective_value = metadata.json()['outputs'][
'{}.objective_value'.format(workflow_name)]
return objective_value
except Exception as e:
logger.info(e)
logger.info(
'Cromwell exception during metadata retrieval, returning bad value...'
)
return bad_value
def get_cromwell_auth(url):
# Provides cromwell authentication to be consumed by all API functions
# Right now we only implement default authorization (no auth other than server URL)
# Can expand upon this later but for now it really doesn't matter
return CromwellAuth.harmonize_credentials(url=url)
