def test_martha_v2_response_without_gs_uri(self):
resp_json = mock.MagicMock(
return_value=self.mock_martha_v2_response_without_gs_uri)
requests_post = mock.MagicMock(
return_value=mock.MagicMock(status_code=200, json=resp_json))
with ExitStack() as es:
es.enter_context(
mock.patch("terra_notebook_utils.drs.gs.get_client"))
es.enter_context(
mock.patch("terra_notebook_utils.drs.http",
post=requests_post))
with self.assertRaisesRegex(
Exception,
f"No GS url found for DRS uri '{self.drs_url}'"):
drs.resolve_drs_for_gs_storage(self.drs_url)
def test_martha_v3_empty_error_response(self):
resp_json = mock.MagicMock(
return_value=self.mock_martha_v3_empty_error_response)
requests_post = mock.MagicMock(
return_value=mock.MagicMock(status_code=500, json=resp_json))
with ExitStack() as es:
es.enter_context(
mock.patch("terra_notebook_utils.drs.gs.get_client"))
es.enter_context(
mock.patch("terra_notebook_utils.drs.http",
post=requests_post))
with self.assertRaisesRegex(
DRSResolutionError,
"Unexpected response while resolving DRS path. Expected status 200, got 500. "
):
drs.resolve_drs_for_gs_storage(self.jade_dev_url)
def test_resolve_drs_for_google_storage(self):
_, info = drs.resolve_drs_for_gs_storage(self.jade_dev_url)
self.assertEqual(info.bucket_name, "broad-jade-dev-data-bucket")
self.assertEqual(
info.key,
"ca8edd48-e954-4c20-b911-b017fedffb67/c0e40912-8b14-43f6-9a2f-b278144d0060"
)
self.assertEqual(info.name, "hapmap_3.3.hg38.vcf.gz")
self.assertEqual(info.size, 62043448)
def _get_blob(path: str, google_project: str) -> google.cloud.storage.blob:
if path.startswith("gs://"):
from terra_notebook_utils import gs
path = path.split("gs://", 1)[1]
bucket_name, key = path.split("/", 1)
blob = gs.get_client(project=google_project).bucket(bucket_name).get_blob(key)
elif path.startswith("drs://"):
from terra_notebook_utils import drs
client, info = drs.resolve_drs_for_gs_storage(path)
blob = client.bucket(info.bucket_name, user_project=google_project).get_blob(info.key)
else:
blob = None
return blob
def _get_fileobj(uri: str):
if uri.startswith("gs://"):
bucket_name, key = uri[5:].split("/", 1)
blob = gs.get_client().bucket(
bucket_name, user_project=WORKSPACE_GOOGLE_PROJECT).get_blob(key)
fh = gscio.Reader(blob, chunk_size=1024**2)
elif uri.startswith("drs://"):
gs_client, drs_info = drs.resolve_drs_for_gs_storage(uri)
bucket = gs_client.bucket(drs_info.bucket_name,
user_project=WORKSPACE_GOOGLE_PROJECT)
fh = gscio.Reader(bucket.get_blob(drs_info.key), chunk_size=1024**2)
else:
fh = open(uri, "rb")
return fh
def test_martha_v2_response(self):
resp_json = mock.MagicMock(return_value=self.mock_martha_v2_response)
requests_post = mock.MagicMock(
return_value=mock.MagicMock(status_code=200, json=resp_json))
with ExitStack() as es:
es.enter_context(
mock.patch("terra_notebook_utils.drs.gs.get_client"))
es.enter_context(
mock.patch("terra_notebook_utils.drs.http",
post=requests_post))
_, actual_info = drs.resolve_drs_for_gs_storage(self.drs_url)
self.assertEqual({'project_id': "foo"}, actual_info.credentials)
self.assertEqual('bogus', actual_info.bucket_name)
self.assertEqual('my_data', actual_info.key)
self.assertEqual('my_data', actual_info.name)
self.assertEqual(15601108255, actual_info.size)
self.assertEqual('2020-04-27T15:56:09.696Z', actual_info.updated)
def test_martha_v2_response_with_missing_fields(self):
resp_json = mock.MagicMock(
return_value=self.mock_martha_v2_response_missing_fields)
requests_post = mock.MagicMock(
return_value=mock.MagicMock(status_code=200, json=resp_json))
with ExitStack() as es:
es.enter_context(
mock.patch("terra_notebook_utils.drs.gs.get_client"))
es.enter_context(
mock.patch("terra_notebook_utils.drs.http",
post=requests_post))
_, actual_info = drs.resolve_drs_for_gs_storage(self.drs_url)
self.assertEqual(None, actual_info.credentials)
self.assertEqual('bogus', actual_info.bucket_name)
self.assertEqual('my_data', actual_info.key)
self.assertEqual(None, actual_info.name)
self.assertEqual(None, actual_info.size)
self.assertEqual(None, actual_info.updated)
def vcf_info(
uri: str,
workspace_namespace: Optional[str] = WORKSPACE_GOOGLE_PROJECT
) -> VCFInfo:
if uri.startswith("drs://"):
client, drs_info = drs.resolve_drs_for_gs_storage(uri)
blob = client.bucket(drs_info.bucket_name,
user_project=workspace_namespace).get_blob(
drs_info.key)
return VCFInfo.with_blob(blob)
elif uri.startswith("gs://"):
bucket, key = uri[5:].split("/", 1)
blob = gs.get_client().bucket(
bucket, user_project=workspace_namespace).get_blob(key)
return VCFInfo.with_blob(blob)
elif uri.startswith("s3://"):
raise ValueError("S3 URIs not supported")
else:
return VCFInfo.with_file(uri)
def test_martha_v3_response_with_missing_fields(self):
resp_json = mock.MagicMock(
return_value=self.mock_martha_v3_response_missing_fields)
requests_post = mock.MagicMock(
return_value=mock.MagicMock(status_code=200, json=resp_json))
with ExitStack() as es:
es.enter_context(
mock.patch("terra_notebook_utils.drs.gs.get_client"))
es.enter_context(
mock.patch("terra_notebook_utils.drs.http",
post=requests_post))
_, actual_info = drs.resolve_drs_for_gs_storage(self.jade_dev_url)
self.assertEqual({'project_id': "foo"}, actual_info.credentials)
self.assertEqual('broad-jade-dev-data-bucket',
actual_info.bucket_name)
self.assertEqual(
'fd8d8492-ad02-447d-b54e-35a7ffd0e7a5/8b07563a-542f-4b5c-9e00-e8fe6b1861de',
actual_info.key)
self.assertEqual(None, actual_info.name)
self.assertEqual(None, actual_info.size)
self.assertEqual(None, actual_info.updated)
def test_martha_v3_response(self):
resp_json = mock.MagicMock(return_value=self.mock_jdr_response)
requests_post = mock.MagicMock(
return_value=mock.MagicMock(status_code=200, json=resp_json))
with ExitStack() as es:
es.enter_context(
mock.patch("terra_notebook_utils.drs.gs.get_client"))
es.enter_context(
mock.patch("terra_notebook_utils.drs.http",
post=requests_post))
_, actual_info = drs.resolve_drs_for_gs_storage(self.jade_dev_url)
self.assertEqual(None, actual_info.credentials)
self.assertEqual('broad-jade-dev-data-bucket',
actual_info.bucket_name)
self.assertEqual(
'fd8d8492-ad02-447d-b54e-35a7ffd0e7a5/8b07563a-542f-4b5c-9e00-e8fe6b1861de',
actual_info.key)
self.assertEqual(
'HG00096.mapped.ILLUMINA.bwa.GBR.low_coverage.20120522.bam',
actual_info.name)
self.assertEqual(15601108255, actual_info.size)
self.assertEqual('2020-04-27T15:56:09.696Z', actual_info.updated)
def test_resolve_drs_for_google_storage(self):
_, info = drs.resolve_drs_for_gs_storage(self.drs_url)
self.assertEqual(info.bucket_name, "topmed-irc-share")
self.assertEqual(info.key, "genomes/NWD522743.b38.irc.v1.cram.crai")
