def test_list_visible_columns_for_schemas(list_visible_columns_item, registry): types = registry[TYPES] doc_types = ['Experiment'] schemas = [types[doc_type].schema for doc_type in doc_types] for request in list_visible_columns_item: columns = list_visible_columns_for_schemas(request, schemas) for param, value in request.params.items(): if param != 'type': assert (value in columns) is True assert isinstance(columns, dict) is True
def report_download(context, request): downloadtime = datetime.datetime.now() types = request.params.getall('type') if len(types) != 1: msg = 'Report view requires specifying a single type.' raise HTTPBadRequest(explanation=msg) # Make sure we get all results request.GET['limit'] = 'all' type_str = types[0] schemas = [request.registry[TYPES][type_str].schema] columns = list_visible_columns_for_schemas(request, schemas) snake_type = _convert_camel_to_snake(type_str).replace("'", '') def format_header(seq): newheader="%s\t%s%s?%s\r\n" % (downloadtime, request.host_url, '/report/', request.query_string) return(bytes(newheader, 'utf-8')) # Work around Excel bug; can't open single column TSV with 'ID' header if len(columns) == 1 and '@id' in columns: columns['@id']['title'] = 'id' header = [column.get('title') or field for field, column in columns.items()] def generate_rows(): yield format_header(header) yield format_row(header) for item in search(context, request, return_generator=True): values = [lookup_column_value(item, path) for path in columns] yield format_row(values) # Stream response using chunked encoding. request.response.content_type = 'text/tsv' request.response.content_disposition = 'attachment;filename="{}_report_{}_{}_{}_{}h_{}m.tsv"'.format( snake_type, downloadtime.year, downloadtime.month, downloadtime.day, downloadtime.hour, downloadtime.minute ) request.response.app_iter = generate_rows() return request.response
def report_download(context, request): downloadtime = datetime.datetime.now() types = request.params.getall('type') if len(types) != 1: msg = 'Report view requires specifying a single type.' raise HTTPBadRequest(explanation=msg) # Make sure we get all results request.GET['limit'] = 'all' type_str = types[0] schemas = [request.registry[TYPES][type_str].schema] columns = list_visible_columns_for_schemas(request, schemas) snake_type = _convert_camel_to_snake(type_str).replace("'", '') def format_header(seq): newheader="%s\t%s%s?%s\r\n" % (downloadtime, request.host_url, '/report/', request.query_string) return(bytes(newheader, 'utf-8')) # Work around Excel bug; can't open single column TSV with 'ID' header if len(columns) == 1 and '@id' in columns: columns['@id']['title'] = 'id' header = [column.get('title') or field for field, column in columns.items()] def generate_rows(): yield format_header(header) yield format_row(header) for item in search(context, request, return_generator=True): values = [lookup_column_value(item, path) for path in columns] yield format_row(values) # Stream response using chunked encoding. request.response.content_type = 'text/tsv' request.response.content_disposition = 'attachment;filename="{}_report_{}_{}_{}_{}h_{}m.tsv"'.format( snake_type, downloadtime.year, downloadtime.month, downloadtime.day, downloadtime.hour, downloadtime.minute ) request.response.app_iter = generate_rows() return request.response
def preprocess_view(self, views=None, search_result_actions=None): # pylint: disable=too-many-statements, too-many-branches, too-many-locals ''' Main function to construct query and build view results json * Only publicly accessible function ''' types = self._types search_base = normalize_query(self._request) result = { '@context': self._request.route_path('jsonld_context'), '@id': '/search/' + search_base, '@type': ['Search'], 'title': 'Search', 'filters': [], } es_index = RESOURCES_INDEX search_audit = self._request.has_permission('search_audit') from_, size = get_pagination(self._request) search_term = prepare_search_term(self._request) if (hasattr(self._context, 'type_info') and hasattr(self._context.type_info, 'name') and self._context.type_info.name): doc_types = [self._context.type_info.name] else: doc_types = self._request.params.getall('type') if '*' in doc_types: doc_types = ['Item'] # Normalize to item_type try: doc_types = sorted({types[name].name for name in doc_types}) except KeyError: # Check for invalid types bad_types = [t for t in doc_types if t not in types] msg = "Invalid type: {}".format(', '.join(bad_types)) raise HTTPBadRequest(explanation=msg) searchterm_specs = self._request.params.getall('searchTerm') searchterm_only = urlencode([("searchTerm", searchterm) for searchterm in searchterm_specs]) if searchterm_only: clear_qs = searchterm_only else: clear_qs = urlencode([("type", typ) for typ in doc_types]) search_route = self._request.route_path('search', slash='/') clear_route = '?' + clear_qs if clear_qs else '' result['clear_filters'] = search_route + clear_route if not doc_types: if self._request.params.get('mode') == 'picker': doc_types = ['Item'] else: doc_types = self._default_doc_types else: for item_type in doc_types: t_thing = types[item_type] q_thing = urlencode([ (k.encode('utf-8'), v.encode('utf-8')) for k, v in self._request.params.items() if not (k == 'type' and types['Item' if v == '*' else v] is t_thing) ]) result['filters'].append({ 'field': 'type', 'term': t_thing.name, 'remove': '{}?{}'.format(self._request.path, q_thing) }) if views: result['views'] = views search_fields, _ = get_search_fields(self._request, doc_types) query = get_filtered_query( search_term, search_fields, sorted(list_result_fields(self._request, doc_types)), self._principals, doc_types, ) schemas = [types[doc_type].schema for doc_type in doc_types] columns = list_visible_columns_for_schemas(self._request, schemas) if columns: result['columns'] = columns if search_term == '*': del query['query']['query_string'] else: query['query']['query_string']['fields'].extend( ['_all', '*.uuid', '*.md5sum', '*.submitted_file_name']) set_sort_order(self._request, search_term, types, doc_types, query, result) used_filters = set_filters(self._request, query, result) facets = [ ('type', { 'title': 'Data Type' }), ] if len(doc_types) == 1 and 'facets' in types[doc_types[0]].schema: facets.extend(types[doc_types[0]].schema['facets'].items()) for audit_facet in self._audit_facets: if (search_audit and 'group.submitter' in self._principals or 'INTERNAL_ACTION' not in audit_facet[0]): facets.append(audit_facet) query['aggs'] = set_facets(facets, used_filters, self._principals, doc_types) query = sort_query(query) do_scan = size is None or size > 1000 if not self._request.params.get('type') or 'Item' in doc_types: es_index = RESOURCES_INDEX else: es_index = [ types[type_name].item_type for type_name in doc_types if hasattr(types[type_name], 'item_type') ] if do_scan: es_results = self._elastic_search.search( body=query, index=es_index, search_type='query_then_fetch') else: es_results = self._elastic_search.search(body=query, index=es_index, from_=from_, size=size, request_cache=True) total = es_results['hits']['total'] result['total'] = total schemas = (types[item_type].schema for item_type in doc_types) result['facets'] = self._format_facets(es_results, facets, used_filters, schemas, total, self._principals) if search_result_actions: result.update( search_result_actions(self._request, doc_types, es_results)) if size is not None and size < result['total']: params = [(k, v) for k, v in self._request.params.items() if k != 'limit'] params.append(('limit', 'all')) result['all'] = '%s?%s' % (self._request.resource_path( self._context), urlencode(params)) if not result['total']: self._request.response.status_code = 404 result['notification'] = 'No results found' result['@graph'] = [] return result if not self._return_generator else [] result['notification'] = 'Success' if not do_scan: graph = format_results(self._request, es_results['hits']['hits'], result) if self._return_generator: return graph result['@graph'] = list(graph) return result del query['aggs'] if size is None: hits = scan(self._elastic_search, query=query, index=es_index, preserve_order=False) else: hits = scan(self._elastic_search, query=query, index=es_index, from_=from_, size=size, preserve_order=False) graph = format_results(self._request, hits, result) if self._request.__parent__ is not None or self._return_generator: if self._return_generator: return graph result['@graph'] = list(graph) return result app_iter = iter_long_json('@graph', graph, result) self._request.response.content_type = 'application/json' if str is bytes: # Python 2 vs 3 wsgi differences self._request.response.app_iter = app_iter # Python 2 else: self._request.response.app_iter = (item.encode('utf-8') for item in app_iter) return self._request.response
def preprocess_view(self, views=None, search_result_actions=None): # pylint: disable=too-many-statements, too-many-branches, too-many-locals ''' Main function to construct query and build view results json * Only publicly accessible function ''' types = self._types search_base = normalize_query(self._request) result = { '@context': self._request.route_path('jsonld_context'), '@id': '/search/' + search_base, '@type': ['Search'], 'title': 'Search', 'filters': [], } es_index = RESOURCES_INDEX search_audit = self._request.has_permission('search_audit') from_, size = get_pagination(self._request) search_term = prepare_search_term(self._request) if ( hasattr(self._context, 'type_info') and hasattr(self._context.type_info, 'name') and self._context.type_info.name ): doc_types = [self._context.type_info.name] else: doc_types = self._request.params.getall('type') if '*' in doc_types: doc_types = ['Item'] # Normalize to item_type try: doc_types = sorted({types[name].name for name in doc_types}) except KeyError: # Check for invalid types bad_types = [t for t in doc_types if t not in types] msg = "Invalid type: {}".format(', '.join(bad_types)) raise HTTPBadRequest(explanation=msg) searchterm_specs = self._request.params.getall('searchTerm') searchterm_only = urlencode( [ ("searchTerm", searchterm) for searchterm in searchterm_specs ] ) if searchterm_only: clear_qs = searchterm_only else: clear_qs = urlencode([("type", typ) for typ in doc_types]) search_route = self._request.route_path('search', slash='/') clear_route = '?' + clear_qs if clear_qs else '' result['clear_filters'] = search_route + clear_route if not doc_types: if self._request.params.get('mode') == 'picker': doc_types = ['Item'] else: doc_types = self._default_doc_types else: for item_type in doc_types: t_thing = types[item_type] q_thing = urlencode( [ (k.encode('utf-8'), v.encode('utf-8')) for k, v in self._request.params.items() if not (k == 'type' and types['Item' if v == '*' else v] is t_thing) ] ) result['filters'].append({ 'field': 'type', 'term': t_thing.name, 'remove': '{}?{}'.format(self._request.path, q_thing) }) if views: result['views'] = views search_fields, _ = get_search_fields(self._request, doc_types) query = get_filtered_query( search_term, search_fields, sorted(list_result_fields(self._request, doc_types)), self._principals, doc_types, ) schemas = [types[doc_type].schema for doc_type in doc_types] columns = list_visible_columns_for_schemas(self._request, schemas) if columns: result['columns'] = columns if search_term == '*': del query['query']['query_string'] else: query['query']['query_string']['fields'].extend( ['_all', '*.uuid', '*.md5sum', '*.submitted_file_name'] ) set_sort_order(self._request, search_term, types, doc_types, query, result) used_filters = set_filters(self._request, query, result) facets = [ ('type', {'title': 'Data Type'}), ] if len(doc_types) == 1 and 'facets' in types[doc_types[0]].schema: facets.extend(types[doc_types[0]].schema['facets'].items()) for audit_facet in self._audit_facets: if ( search_audit and 'group.submitter' in self._principals or 'INTERNAL_ACTION' not in audit_facet[0] ): facets.append(audit_facet) query['aggs'] = set_facets(facets, used_filters, self._principals, doc_types) query = sort_query(query) do_scan = size is None or size > 1000 if not self._request.params.get('type') or 'Item' in doc_types: es_index = RESOURCES_INDEX else: es_index = [ types[type_name].item_type for type_name in doc_types if hasattr(types[type_name], 'item_type') ] if do_scan: es_results = self._elastic_search.search( body=query, index=es_index, search_type='query_then_fetch' ) else: es_results = self._elastic_search.search( body=query, index=es_index, from_=from_, size=size, request_cache=True ) total = es_results['hits']['total'] result['total'] = total schemas = (types[item_type].schema for item_type in doc_types) result['facets'] = self._format_facets( es_results, facets, used_filters, schemas, total, self._principals ) if search_result_actions: result.update( search_result_actions( self._request, doc_types, es_results ) ) if size is not None and size < result['total']: params = [(k, v) for k, v in self._request.params.items() if k != 'limit'] params.append(('limit', 'all')) result['all'] = '%s?%s' % ( self._request.resource_path(self._context), urlencode(params) ) if not result['total']: self._request.response.status_code = 404 result['notification'] = 'No results found' result['@graph'] = [] return result if not self._return_generator else [] result['notification'] = 'Success' if not do_scan: graph = format_results( self._request, es_results['hits']['hits'], result ) if self._return_generator: return graph result['@graph'] = list(graph) return result del query['aggs'] if size is None: hits = scan( self._elastic_search, query=query, index=es_index, preserve_order=False ) else: hits = scan( self._elastic_search, query=query, index=es_index, from_=from_, size=size, preserve_order=False ) graph = format_results(self._request, hits, result) if self._request.__parent__ is not None or self._return_generator: if self._return_generator: return graph result['@graph'] = list(graph) return result app_iter = iter_long_json('@graph', graph, result) self._request.response.content_type = 'application/json' if str is bytes: # Python 2 vs 3 wsgi differences self._request.response.app_iter = app_iter # Python 2 else: self._request.response.app_iter = ( item.encode('utf-8') for item in app_iter ) return self._request.response