コード例 #1
0
ファイル: drs.py プロジェクト: mitchac/terra-notebook-utils
def fetch_drs_info(drs_url: str) -> dict:
    """
    Request DRS infromation from martha.
    """
    access_token = gs.get_access_token()

    headers = {
        'authorization': f"Bearer {access_token}",
        'content-type': "application/json"
    }

    logger.info(f"Resolving DRS uri '{drs_url}' through '{MARTHA_URL}'.")

    json_body = dict(url=drs_url)
    resp = http.post(MARTHA_URL, headers=headers, json=json_body)

    if 200 == resp.status_code:
        resp_data = resp.json()
    else:
        logger.warning(resp.content)
        response_json = resp.json()

        if 'response' in response_json:
            if 'text' in response_json['response']:
                error_details = f"Error: {response_json['response']['text']}"
            else:
                error_details = ""
        else:
            error_details = ""

        raise DRSResolutionError(f"Unexpected response while resolving DRS path. Expected status 200, got "
                                 f"{resp.status_code}. {error_details}")

    return resp_data
コード例 #2
0
def enable_requester_pays(
        workspace_name: Optional[str] = WORKSPACE_NAME,
        workspace_namespace: Optional[str] = WORKSPACE_NAMESPACE):
    assert workspace_name
    import urllib.parse
    encoded_workspace = urllib.parse.quote(workspace_name)
    rawls_url = (
        f"https://rawls.dsde-{TERRA_DEPLOYMENT_ENV}.broadinstitute.org/api/workspaces/"
        f"{workspace_namespace}/{encoded_workspace}/enableRequesterPaysForLinkedServiceAccounts"
    )
    logger.info(
        "Enabling requester pays for your workspace. This will only take a few seconds..."
    )
    access_token = gs.get_access_token()

    headers = {
        'authorization': f"Bearer {access_token}",
        'content-type': "application/json"
    }
    resp = http.put(rawls_url, headers=headers)

    if resp.status_code != 204:
        logger.warning(
            f"Failed to init requester pays for workspace {workspace_namespace}/{workspace_name}: "
            f"Expected '204', got '{resp.status_code}' for '{rawls_url}'. "
            "You will not be able to access DRS URIs that interact with requester pays buckets."
        )
コード例 #3
0
def get_drs(drs_url: str) -> Response:
    """Request DRS information from martha."""
    access_token = gs.get_access_token()

    headers = {
        'authorization': f"Bearer {access_token}",
        'content-type': "application/json"
    }

    logger.debug(f"Resolving DRS uri '{drs_url}' through '{MARTHA_URL}'.")

    json_body = dict(url=drs_url,
                     fields=[
                         "fileName", "hashes", "size", "gsUri", "bucket",
                         "name", "timeUpdated", "googleServiceAccount",
                         "accessUrl"
                     ])
    resp = http.post(MARTHA_URL, headers=headers, json=json_body)

    if 200 != resp.status_code:
        logger.warning(resp.content)
        error_details = resp.json().get('response', {}).get('text', '')
        raise DRSResolutionError(
            f"Unexpected response while resolving DRS URI. Expected status 200, got "
            f"{resp.status_code}. {error_details}")

    return resp
コード例 #4
0
def import_dockstore_wf_into_terra():
    workspace = 'BDC_Dockstore_Import_Test'
    endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/methodconfigs'

    token = gs.get_access_token()
    headers = {
        'Content-Type': 'application/json',
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }

    data = {
        "namespace": BILLING_PROJECT,
        "name": "UM_aligner_wdl",
        "rootEntityType": "",
        "inputs": {},
        "outputs": {},
        "prerequisites": {},
        "methodRepoMethod": {
            "sourceRepo": "dockstore",
            "methodPath":
            "github.com/DataBiosphere/topmed-workflows/UM_aligner_wdl",
            "methodVersion": "1.32.0"
        },
        "methodConfigVersion": 1,
        "deleted": False
    }

    resp = requests.post(endpoint, headers=headers, data=json.dumps(data))
    resp.raise_for_status()
    return resp.json()
コード例 #5
0
def run_workflow():
    workspace = 'DRS-Test-Workspace'
    endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/submissions'

    token = gs.get_access_token()
    headers = {
        'Content-Type': 'application/json',
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }

    data = {
        "methodConfigurationNamespace": "drs_tests",
        "methodConfigurationName": "md5sum",
        "entityType": "data_access_test_drs_uris_set",
        "entityName": "md5sum_2020-05-19T17-52-42",
        "expression": "this.data_access_test_drs_uriss",
        "useCallCache": False,
        "deleteIntermediateOutputFiles": True,
        "workflowFailureMode": "NoNewCalls"
    }

    resp = requests.post(endpoint, headers=headers, data=json.dumps(data))
    resp.raise_for_status()
    return resp.json()
コード例 #6
0
def delete_terra_workspace(workspace):
    endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}'

    token = gs.get_access_token()
    headers = {'Accept': 'text/plain', 'Authorization': f'Bearer {token}'}

    resp = requests.delete(endpoint, headers=headers)

    return resp
コード例 #7
0
def check_workflow_status(submission_id):
    workspace = 'DRS-Test-Workspace'
    endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/submissions/{submission_id}'

    token = gs.get_access_token()
    headers = {
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }

    resp = requests.get(endpoint, headers=headers)
    resp.raise_for_status()
    return resp.json()
コード例 #8
0
def check_workflow_presence_in_terra_workspace():
    workspace = 'BDC_Dockstore_Import_Test'
    endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/methodconfigs?allRepos=true'

    token = gs.get_access_token()
    headers = {
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }

    resp = requests.get(endpoint, headers=headers)
    resp.raise_for_status()
    return resp.json()
コード例 #9
0
def import_drs_from_gen3(guid: str) -> requests.Response:
    """
    Import the first byte of a DRS URI using gen3.

    Makes two calls, first one to gen3, which returns the link needed to make the second
    call to the google API and fetch directly from the google bucket.
    """
    if guid.startswith('drs://'):
        guid = guid[len('drs://'):]
    else:
        raise ValueError(
            f'DRS URI is missing the "drs://" schema.  Please specify a DRS URI, not: {guid}'
        )
    gen3_endpoint = f'{GEN3_DOMAIN}/user/data/download/{guid}'
    token = gs.get_access_token()
    headers = {
        'Content-Type': 'application/json',
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }
    gen3_resp = requests.get(gen3_endpoint, headers=headers)

    if gen3_resp.ok:
        # Example of the url that gen3 returns:
        #   google_uri = 'https://storage.googleapis.com/fc-56ac46ea-efc4-4683-b6d5-6d95bed41c5e/CCDG_13607/Project_CCDG_13607_B01_GRM_WGS.gVCF.2019-02-06/Sample_HG03611/analysis/HG03611.haplotypeCalls.er.raw.g.vcf.gz'
        #   access_id_arg = '[email protected]'
        #   expires_arg = 'Expires=1607381713'
        #   signature_arg = 'Signature=hugehashofmanycharsincluding%=='
        #   endpoint_looks_like = f'{google_uri}?{access_id_arg}&{expires_arg}&{signature_arg}'
        gs_endpoint = gen3_resp.json()["url"]
        gs_endpoint_w_requester_pays = add_requester_pays_arg_to_url(
            gs_endpoint)

        # Use 'Range' header to only download the first two bytes
        # https://cloud.google.com/storage/docs/json_api/v1/parameters#range
        headers['Range'] = 'bytes=0-1'

        gs_resp = requests.get(gs_endpoint_w_requester_pays, headers=headers)
        if gs_resp.ok:
            return gs_resp
        else:
            print(
                f'Gen3 url call succeeded for: {gen3_endpoint} with: {gen3_resp.json()} ...\n'
                f'BUT the subsequent google called failed: {gs_endpoint_w_requester_pays} with: {gs_resp.content}'
            )
            gs_resp.raise_for_status()
    else:
        print(
            f'Gen3 url call failed for: {gen3_endpoint} with: {gen3_resp.content}'
        )
        gen3_resp.raise_for_status()
コード例 #10
0
def delete_workflow_presence_in_terra_workspace():
    workspace = 'BDC_Dockstore_Import_Test'
    workflow = 'UM_aligner_wdl'
    endpoint = f'{RAWLS_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/methodconfigs/{BILLING_PROJECT}/{workflow}'

    token = gs.get_access_token()
    headers = {
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }

    resp = requests.delete(endpoint, headers=headers)
    resp.raise_for_status()
    return {}
コード例 #11
0
def pfb_job_status_in_terra(workspace, job_id):
    endpoint = f'{ORC_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/importPFB/{job_id}'
    token = gs.get_access_token()

    headers = {
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }

    resp = requests.get(endpoint, headers=headers)

    if resp.ok:
        return resp.json()
    else:
        print(resp.content)
        resp.raise_for_status()
コード例 #12
0
def import_pfb(workspace):
    endpoint = f'{ORC_DOMAIN}/api/workspaces/{BILLING_PROJECT}/{workspace}/importPFB'

    token = gs.get_access_token()
    headers = {
        'Content-Type': 'application/json',
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }
    data = dict(
        url=
        'https://cdistest-public-test-bucket.s3.amazonaws.com/export_2020-06-02T17_33_36.avro'
    )

    resp = requests.post(endpoint, headers=headers, data=json.dumps(data))

    if resp.ok:
        return resp.json()
    else:
        print(resp.content)
        resp.raise_for_status()
コード例 #13
0
def create_terra_workspace(workspace):
    endpoint = f'{RAWLS_DOMAIN}/api/workspaces'

    token = gs.get_access_token()
    headers = {
        'Content-Type': 'application/json',
        'Accept': 'application/json',
        'Authorization': f'Bearer {token}'
    }

    data = dict(namespace=BILLING_PROJECT,
                name=workspace,
                authorizationDomain=[],
                attributes={'description': ''},
                copyFilesWithPrefix='notebooks/')

    resp = requests.post(endpoint, headers=headers, data=json.dumps(data))

    if resp.ok:
        return resp.json()
    else:
        print(resp.content)
        resp.raise_for_status()
コード例 #14
0
 def test_get_access_token(self):
     gs.get_access_token()