def import_dicom_bucket(dicom_dataset_project_id,
dicom_dataset_cloud_region,
dicom_dataset_id,
dicom_store_id,
bucket_project_id,
bucket_name,
sub_bucket_name=None):
logger = logging.getLogger(__name__)
client = get_client()
dicom_store_parent = "projects/{}/locations/{}/datasets/{}".format(
dicom_dataset_project_id, dicom_dataset_cloud_region, dicom_dataset_id)
dicom_store_name = "{}/dicomStores/{}".format(dicom_store_parent,
dicom_store_id)
sub_bucket = '' if sub_bucket_name is None else '{}/'.format(
sub_bucket_name)
body = {
"gcsSource": {
"uri": "gs://{}/{}**.dcm".format(bucket_name, sub_bucket_name)
}
}
request = (client.projects().locations().datasets().dicomStores().import_(
name=dicom_store_name, body=body))
response = request.execute()
n = 0
request = (client.projects().locations().datasets().operations().get(
name=response['name']))
request_interval = 10
tic = time.time()
while True:
time.sleep(request_interval)
n += 1
toc = time.time()
time_elapsed = toc - tic
logger.info('waiting for bucket import operation to '
'finish -> elapsed time: {}'.format(
timedelta(seconds=time_elapsed)))
res = request.execute()
if 'done' in res:
logger.debug(ct.dict2str(res, 1, '\t'))
if res['done'] == True:
break
# r = ct.dict2str(res, 1, ' ')
# logger.info("-->{}) {}".format(n, r))
success: bool = False
if 'response' in res:
logger.info(
"DICOM bucket {} imported successfully to dicom store {}".format(
bucket_name, dicom_store_name))
elif 'error' in res:
logger.info('DICOM bucket {} failed to be imported -> error {}'.format(
bucket_name, ct.dict2str(res['error'], 1, '\t')))
else:
logger.info('The DICOM bucket {} import operation finished'
' but its not clear it was successful'.format(bucket_name))
return success
def export_dicom_instance_bigquery(dicomstore_project_id,
dicomstore_cloud_region,
dicomstore_dataset_id, dicom_store_id,
bigquery_project_id, bigquery_dataset_id,
bigquery_table_id):
logger = logging.getLogger(__name__)
client = get_client()
dicom_store_parent = "projects/{}/locations/{}/datasets/{}".format(
dicomstore_project_id, dicomstore_cloud_region, dicomstore_dataset_id)
dicom_store_name = "{}/dicomStores/{}".format(dicom_store_parent,
dicom_store_id)
uri_prefix = '{}.{}.{}'.format(dicomstore_project_id, bigquery_dataset_id,
bigquery_table_id)
body = {"bigqueryDestination": {"tableUri": "bq://{}".format(uri_prefix)}}
request = (client.projects().locations().datasets().dicomStores().export(
name=dicom_store_name, body=body))
response = request.execute()
n = 0
request = (client.projects().locations().datasets().operations().get(
name=response['name']))
request_interval = 10
tic = time.time()
while True:
time.sleep(request_interval)
n += 1
toc = time.time()
time_elapsed = toc - tic
logger.info('waiting for bucket export operation to '
'finish -> elapsed time: {}'.format(
timedelta(seconds=time_elapsed)))
res = request.execute()
if 'done' in res:
logger.debug(ct.dict2str(res, 1, '\t'))
if res['done'] == True:
break
# r = ct.dict2str(res, 1, ' ')
# logger.info("-->{}) {}".format(n, r))
success: bool = False
if 'response' in res:
logger.info("DICOM instance {} exported successfully to storage "
"bucket {}".format(dicom_store_id, uri_prefix))
elif 'error' in res:
logger.info(
'DICOM instance {} failed to be exported -> error {}'.format(
dicom_store_id, ct.dict2str(res['error'], 1, '\t')))
else:
logger.info(
'The DICOM instance {} export operation finished'
' but its not clear it was successful'.format(dicom_store_id))
return success
progress.start()
collection_query = """
SELECT DISTINCT GCS_BUCKET FROM
`{}`
ORDER BY GCS_BUCKET
""".format('canceridc-data.idc_views.dicom_all')
q_results = query_string_with_result(collection_query,
project_name='idc-tcia')
all_sizes = {}
whole_siz = 0.0
pss = pl.ProcessPool(16, 'size')
for row in q_results:
bucket_name = row.GCS_BUCKET
pss.queue.put((
get_bucket_size,
(
'idc-tcia',
bucket_name,
),
))
pss.queue.join()
pss.kill_them_all()
progress.kill_timer()
for args, (bucket_name, b_size) in pss.output:
all_sizes[bucket_name] = (b_size, ct.get_human_readable_string(b_size))
print(ct.get_human_readable_string(b_size))
whole_siz += b_size
print(whole_siz)
all_sizes['whole'] = (whole_siz, ct.get_human_readable_string(whole_siz))
print(ct.dict2str(all_sizes))