def batch_download_factory(context, request):
qs = QueryString(request)
specified_type = qs.get_one_value(params=qs.get_type_filters())
if specified_type == 'PublicationData':
return _get_publication_data_batch_download(context, request)
else:
return _get_batch_download(context, request)
def metadata_report_factory(context, request):
qs = QueryString(request)
specified_type = qs.get_one_value(params=qs.get_type_filters())
if specified_type == 'Annotation':
return _get_annotation_metadata(context, request)
elif specified_type == 'PublicationData':
return _get_publication_data_metadata(context, request)
else:
return _get_metadata(context, request)
def batch_download(context, request):
default_params = [
('limit', 'all'),
('field', 'files.href'),
('field', 'files.restricted'),
('field', 'files.file_format'),
('field', 'files.file_format_type'),
('field', 'files.status'),
('field', 'files.assembly'),
]
qs = QueryString(request)
param_list = qs.group_values_by_key()
file_filters = qs.param_keys_to_list(params=qs.get_filters_by_condition(
key_and_value_condition=lambda k, _: k.startswith('files.')))
# Process PublicationData batch downloads separately.
type_param = param_list.get('type', [''])[0]
if type_param and type_param.lower() == 'publicationdata':
return _batch_download_publicationdata(request)
file_fields = [('field', k) for k in file_filters]
qs.drop('limit')
type_param = param_list.get('type', [''])[0]
cart_uuids = param_list.get('cart', [])
# Only allow specific type= query-string values, or cart=.
if not type_param and not cart_uuids:
raise HTTPBadRequest(
explanation='URL must include a "type" or "cart" parameter.')
if not type_param.lower() in _allowed_types:
raise HTTPBadRequest(explanation='"{}" not a valid type for metadata'.
format(type_param))
# Check for the "visualizable" and/or "raw" options in the query string for file filtering.
visualizable_only = qs.is_param('option', 'visualizable')
raw_only = qs.is_param('option', 'raw')
qs.drop('option')
qs.extend(default_params + file_fields)
experiments = []
if request.method == 'POST':
metadata_link = ''
cart_uuid = qs.get_one_value(params=qs.get_key_filters(key='cart'))
try:
elements = request.json.get('elements', [])
except ValueError:
elements = []
if cart_uuid:
try:
request.embed(cart_uuid, '@@object')
except KeyError:
raise HTTPBadRequest(
explanation='Specified cart does not exist.')
# metadata.tsv link includes a cart UUID
metadata_link = '{host_url}/metadata/?{search_params}'.format(
host_url=request.host_url,
search_params=qs._get_original_query_string())
else:
metadata_link = '{host_url}/metadata/?{search_params} -X GET -H "Accept: text/tsv" -H "Content-Type: application/json" --data \'{{"elements": [{elements_json}]}}\''.format(
host_url=request.host_url,
search_params=qs._get_original_query_string(),
elements_json=','.join('"{0}"'.format(element)
for element in elements))
# Because of potential number of datasets in the cart, break search
# into multiple searches of ELEMENT_CHUNK_SIZE datasets each.
for i in range(0, len(elements), ELEMENT_CHUNK_SIZE):
qs.drop('@id')
qs.extend([('@id', e) for e in elements[i:i + ELEMENT_CHUNK_SIZE]])
path = '/search/?{}'.format(str(qs))
results = request.embed(quote(path), as_user=True)
experiments.extend(results['@graph'])
else:
# Make sure regular batch download doesn't include a cart parameter; error if it does.
if cart_uuids:
raise HTTPBadRequest(
explanation=
'You must download cart file manifests from the portal.')
# Regular batch download has single simple call to request.embed
metadata_link = '{host_url}/metadata/?{search_params}'.format(
host_url=request.host_url,
search_params=qs._get_original_query_string())
path = '/search/?{}'.format(str(qs))
results = request.embed(quote(path), as_user=True)
experiments = results['@graph']
exp_files = (exp_file for exp in experiments
for exp_file in exp.get('files', []))
files = [metadata_link]
param_list = qs.group_values_by_key()
for exp_file in exp_files:
if not files_prop_param_list(exp_file, param_list):
continue
elif visualizable_only and not is_file_visualizable(exp_file):
continue
elif raw_only and exp_file.get('assembly'):
# "raw" option only allows files w/o assembly.
continue
elif restricted_files_present(exp_file):
continue
files.append('{host_url}{href}'.format(
host_url=request.host_url,
href=exp_file['href'],
))
return Response(content_type='text/plain',
body='\n'.join(files),
content_disposition='attachment; filename="%s"' %
'files.txt')
def batch_download(context, request):
default_params = [('limit', 'all'), ('field', 'files.href'),
('field', 'files.restricted')]
qs = QueryString(request)
file_filters = qs.param_keys_to_list(params=qs.get_filters_by_condition(
key_and_value_condition=lambda k, _: k.startswith('files.')))
file_fields = [('field', k) for k in file_filters]
qs.drop('limit')
qs.extend(default_params + file_fields)
experiments = []
error_message = None
if request.method == 'POST':
metadata_link = ''
cart_uuid = qs.get_one_value(params=qs.get_key_filters(key='cart'))
try:
elements = request.json.get('elements', [])
except ValueError:
elements = []
if cart_uuid:
# metadata.tsv link includes a cart UUID
metadata_link = '{host_url}/metadata/?{search_params}'.format(
host_url=request.host_url,
search_params=qs._get_original_query_string())
else:
metadata_link = '{host_url}/metadata/?{search_params} -X GET -H "Accept: text/tsv" -H "Content-Type: application/json" --data \'{{"elements": [{elements_json}]}}\''.format(
host_url=request.host_url,
search_params=qs._get_original_query_string(),
elements_json=','.join('"{0}"'.format(element)
for element in elements))
# Because of potential number of datasets in the cart, break search
# into multiple searches of ELEMENT_CHUNK_SIZE datasets each.
for i in range(0, len(elements), ELEMENT_CHUNK_SIZE):
qs.drop('@id')
qs.extend([('@id', e) for e in elements[i:i + ELEMENT_CHUNK_SIZE]])
path = '/search/?{}'.format(str(qs))
results = request.embed(quote(path), as_user=True)
experiments.extend(results['@graph'])
else:
# Regular batch download has single simple call to request.embed
metadata_link = '{host_url}/metadata/?{search_params}'.format(
host_url=request.host_url,
search_params=qs._get_original_query_string())
path = '/search/?{}'.format(str(qs))
results = request.embed(quote(path), as_user=True)
experiments = results['@graph']
exp_files = (exp_file for exp in experiments
for exp_file in exp.get('files', []))
files = [metadata_link]
param_list = qs.group_values_by_key()
for exp_file in exp_files:
if not files_prop_param_list(exp_file, param_list):
continue
elif restricted_files_present(exp_file):
continue
files.append('{host_url}{href}'.format(
host_url=request.host_url,
href=exp_file['href'],
))
return Response(content_type='text/plain',
body='\n'.join(files),
content_disposition='attachment; filename="%s"' %
'files.txt')