def fetch_drs_info(drs_url: str) -> dict:
"""
Request DRS infromation from martha.
"""
access_token = gs.get_access_token()
headers = {
'authorization': f"Bearer {access_token}",
'content-type': "application/json"
}
logger.info(f"Resolving DRS uri '{drs_url}' through '{MARTHA_URL}'.")
json_body = dict(url=drs_url)
resp = http.post(MARTHA_URL, headers=headers, json=json_body)
if 200 == resp.status_code:
resp_data = resp.json()
else:
logger.warning(resp.content)
response_json = resp.json()
if 'response' in response_json:
if 'text' in response_json['response']:
error_details = f"Error: {response_json['response']['text']}"
else:
error_details = ""
else:
error_details = ""
raise DRSResolutionError(f"Unexpected response while resolving DRS path. Expected status 200, got "
f"{resp.status_code}. {error_details}")
return resp_data
def enable_requester_pays(
workspace_name: Optional[str] = WORKSPACE_NAME,
workspace_namespace: Optional[str] = WORKSPACE_NAMESPACE):
assert workspace_name
import urllib.parse
encoded_workspace = urllib.parse.quote(workspace_name)
rawls_url = (
f"https://rawls.dsde-{TERRA_DEPLOYMENT_ENV}.broadinstitute.org/api/workspaces/"
f"{workspace_namespace}/{encoded_workspace}/enableRequesterPaysForLinkedServiceAccounts"
)
logger.info(
"Enabling requester pays for your workspace. This will only take a few seconds..."
)
access_token = gs.get_access_token()
headers = {
'authorization': f"Bearer {access_token}",
'content-type': "application/json"
}
resp = http.put(rawls_url, headers=headers)
if resp.status_code != 204:
logger.warning(
f"Failed to init requester pays for workspace {workspace_namespace}/{workspace_name}: "
f"Expected '204', got '{resp.status_code}' for '{rawls_url}'. "
"You will not be able to access DRS URIs that interact with requester pays buckets."
)
def get_drs(drs_url: str) -> Response:
"""Request DRS information from martha."""
access_token = gs.get_access_token()
headers = {
'authorization': f"Bearer {access_token}",
'content-type': "application/json"
}
logger.debug(f"Resolving DRS uri '{drs_url}' through '{MARTHA_URL}'.")
json_body = dict(url=drs_url,
fields=[
"fileName", "hashes", "size", "gsUri", "bucket",
"name", "timeUpdated", "googleServiceAccount",
"accessUrl"
])
resp = http.post(MARTHA_URL, headers=headers, json=json_body)
if 200 != resp.status_code:
logger.warning(resp.content)
error_details = resp.json().get('response', {}).get('text', '')
raise DRSResolutionError(
f"Unexpected response while resolving DRS URI. Expected status 200, got "
f"{resp.status_code}. {error_details}")
return resp
def import_dockstore_wf_into_terra():
workspace = 'BDC_Dockstore_Import_Test'
endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/methodconfigs'
token = gs.get_access_token()
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
data = {
"namespace": BILLING_PROJECT,
"name": "UM_aligner_wdl",
"rootEntityType": "",
"inputs": {},
"outputs": {},
"prerequisites": {},
"methodRepoMethod": {
"sourceRepo": "dockstore",
"methodPath":
"github.com/DataBiosphere/topmed-workflows/UM_aligner_wdl",
"methodVersion": "1.32.0"
},
"methodConfigVersion": 1,
"deleted": False
}
resp = requests.post(endpoint, headers=headers, data=json.dumps(data))
resp.raise_for_status()
return resp.json()
def run_workflow():
workspace = 'DRS-Test-Workspace'
endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/submissions'
token = gs.get_access_token()
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
data = {
"methodConfigurationNamespace": "drs_tests",
"methodConfigurationName": "md5sum",
"entityType": "data_access_test_drs_uris_set",
"entityName": "md5sum_2020-05-19T17-52-42",
"expression": "this.data_access_test_drs_uriss",
"useCallCache": False,
"deleteIntermediateOutputFiles": True,
"workflowFailureMode": "NoNewCalls"
}
resp = requests.post(endpoint, headers=headers, data=json.dumps(data))
resp.raise_for_status()
return resp.json()
def delete_terra_workspace(workspace):
endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}'
token = gs.get_access_token()
headers = {'Accept': 'text/plain', 'Authorization': f'Bearer {token}'}
resp = requests.delete(endpoint, headers=headers)
return resp
def check_workflow_status(submission_id):
workspace = 'DRS-Test-Workspace'
endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/submissions/{submission_id}'
token = gs.get_access_token()
headers = {
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
resp = requests.get(endpoint, headers=headers)
resp.raise_for_status()
return resp.json()
def check_workflow_presence_in_terra_workspace():
workspace = 'BDC_Dockstore_Import_Test'
endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/methodconfigs?allRepos=true'
token = gs.get_access_token()
headers = {
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
resp = requests.get(endpoint, headers=headers)
resp.raise_for_status()
return resp.json()
def import_drs_from_gen3(guid: str) -> requests.Response:
"""
Import the first byte of a DRS URI using gen3.
Makes two calls, first one to gen3, which returns the link needed to make the second
call to the google API and fetch directly from the google bucket.
"""
if guid.startswith('drs://'):
guid = guid[len('drs://'):]
else:
raise ValueError(
f'DRS URI is missing the "drs://" schema. Please specify a DRS URI, not: {guid}'
)
gen3_endpoint = f'{GEN3_DOMAIN}/user/data/download/{guid}'
token = gs.get_access_token()
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
gen3_resp = requests.get(gen3_endpoint, headers=headers)
if gen3_resp.ok:
# Example of the url that gen3 returns:
# google_uri = 'https://storage.googleapis.com/fc-56ac46ea-efc4-4683-b6d5-6d95bed41c5e/CCDG_13607/Project_CCDG_13607_B01_GRM_WGS.gVCF.2019-02-06/Sample_HG03611/analysis/HG03611.haplotypeCalls.er.raw.g.vcf.gz'
# access_id_arg = '[email protected]'
# expires_arg = 'Expires=1607381713'
# signature_arg = 'Signature=hugehashofmanycharsincluding%=='
# endpoint_looks_like = f'{google_uri}?{access_id_arg}&{expires_arg}&{signature_arg}'
gs_endpoint = gen3_resp.json()["url"]
gs_endpoint_w_requester_pays = add_requester_pays_arg_to_url(
gs_endpoint)
# Use 'Range' header to only download the first two bytes
# https://cloud.google.com/storage/docs/json_api/v1/parameters#range
headers['Range'] = 'bytes=0-1'
gs_resp = requests.get(gs_endpoint_w_requester_pays, headers=headers)
if gs_resp.ok:
return gs_resp
else:
print(
f'Gen3 url call succeeded for: {gen3_endpoint} with: {gen3_resp.json()} ...\n'
f'BUT the subsequent google called failed: {gs_endpoint_w_requester_pays} with: {gs_resp.content}'
)
gs_resp.raise_for_status()
else:
print(
f'Gen3 url call failed for: {gen3_endpoint} with: {gen3_resp.content}'
)
gen3_resp.raise_for_status()
def delete_workflow_presence_in_terra_workspace():
workspace = 'BDC_Dockstore_Import_Test'
workflow = 'UM_aligner_wdl'
endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/methodconfigs/{BILLING_PROJECT}/{workflow}'
token = gs.get_access_token()
headers = {
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
resp = requests.delete(endpoint, headers=headers)
resp.raise_for_status()
return {}
def pfb_job_status_in_terra(workspace, job_id):
endpoint = f'{ORC_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/importPFB/{job_id}'
token = gs.get_access_token()
headers = {
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
resp = requests.get(endpoint, headers=headers)
if resp.ok:
return resp.json()
else:
print(resp.content)
resp.raise_for_status()
def import_pfb(workspace):
endpoint = f'{ORC_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/importPFB'
token = gs.get_access_token()
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
data = dict(
url=
'https://cdistest-public-test-bucket.s3.amazonaws.com/export_2020-06-02T17_33_36.avro'
)
resp = requests.post(endpoint, headers=headers, data=json.dumps(data))
if resp.ok:
return resp.json()
else:
print(resp.content)
resp.raise_for_status()
def create_terra_workspace(workspace):
endpoint = f'{RAWLS_DOMAIN}/api/workspaces'
token = gs.get_access_token()
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {token}'
}
data = dict(namespace=BILLING_PROJECT,
name=workspace,
authorizationDomain=[],
attributes={'description': ''},
copyFilesWithPrefix='notebooks/')
resp = requests.post(endpoint, headers=headers, data=json.dumps(data))
if resp.ok:
return resp.json()
else:
print(resp.content)
resp.raise_for_status()
def test_get_access_token(self):
gs.get_access_token()