def search():
datasets, organizations, reuses = multiquery(
SearchQuery(DatasetSearch, **multi_to_dict(request.args)),
SearchQuery(OrganizationSearch, **multi_to_dict(request.args)),
SearchQuery(ReuseSearch, **multi_to_dict(request.args)),
)
return render("search.html", datasets=datasets, organizations=organizations, reuses=reuses)
def get(self):
if self.search_adapter:
result = search.query(self.search_adapter, **multi_to_dict(request.args))
objects = result.get_objects()
else:
objects = list(self.model.objects)
return marshal(objects, self.fields)
def redirect_to_lang(*args, **kwargs):
'''Redirect non lang-prefixed urls to default language.'''
endpoint = request.endpoint.replace('_redirect', '')
kwargs = multi_to_dict(request.args)
kwargs.update(request.view_args)
kwargs['lang_code'] = default_lang
return redirect(url_for(endpoint, **kwargs))
def redirect_to_unlocalized(*args, **kwargs):
'''Redirect lang-prefixed urls to no prefixed URL.'''
endpoint = request.endpoint.replace('_redirect', '')
kwargs = multi_to_dict(request.args)
kwargs.update(request.view_args)
kwargs.pop('lang_code', None)
return redirect(url_for(endpoint, **kwargs))
def get(self):
"""List all objects"""
if self.search_adapter:
objects = search.query(self.search_adapter, **multi_to_dict(request.args))
else:
objects = list(self.model.objects)
return marshal_page(objects, self.fields)
def redirect_to_lang(*args, **kwargs):
"""Redirect non lang-prefixed urls to default language."""
endpoint = request.endpoint.replace("_redirect", "")
kwargs = multi_to_dict(request.args)
kwargs.update(request.view_args)
kwargs["lang_code"] = default_lang
return redirect(url_for(endpoint, **kwargs))
def render_search():
params = multi_to_dict(request.args)
params['facets'] = True
# We only fetch relevant data for the given filter.
if 'tag' in params:
search_queries = [
search.SearchQuery(Dataset, **params),
search.SearchQuery(Reuse, **params)
]
results_labels = ['datasets', 'reuses']
elif 'badge' in params:
search_queries = [
search.SearchQuery(Dataset, **params),
search.SearchQuery(Organization, **params)
]
results_labels = ['datasets', 'organizations']
else:
search_queries = [
search.SearchQuery(Dataset, **params),
search.SearchQuery(Reuse, **params),
search.SearchQuery(Organization, **params),
search.SearchQuery(User, **params)
]
results_labels = ['datasets', 'reuses', 'organizations', 'users']
results = search.multiquery(*search_queries)
return theme.render('search.html',
**dict(zip(results_labels, results)))
def rdf_catalog_format(format):
params = multi_to_dict(request.args)
page = int(params.get('page', 1))
page_size = int(params.get('page_size', 100))
datasets = Dataset.objects.visible().paginate(page, page_size)
catalog = build_catalog(current_site, datasets, format=format)
return graph_response(catalog, format)
def datasets(topic):
kwargs = multi_to_dict(request.args)
kwargs.update(topic=topic)
return theme.render('topic/datasets.html',
topic=topic,
datasets=TopicSearchQuery(Dataset, facets=True, **kwargs).execute()
)
def reuses(topic):
kwargs = multi_to_dict(request.args)
kwargs.update(topic=topic)
return theme.render('topic/reuses.html',
topic=topic,
reuses=TopicSearchQuery(Reuse, facets=True, **kwargs).execute()
)
def reuses_csv():
params = multi_to_dict(request.args)
# redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
exported_models = current_app.config.get('EXPORT_CSV_MODELS', [])
if not params and 'reuse' in exported_models:
return redirect(get_export_url('reuse'))
params['facets'] = False
reuses = search.iter(Reuse, **params)
return csv.stream(ReuseCsvAdapter(reuses), 'reuses')
def organizations_csv():
params = multi_to_dict(request.args)
# redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
exported_models = current_app.config.get('EXPORT_CSV_MODELS', [])
if not params and 'organization' in exported_models:
return redirect(get_export_url('organization'))
params['facets'] = False
organizations = search.iter(Organization, **params)
return csv.stream(OrganizationCsvAdapter(organizations), 'organizations')
def datasets_csv():
params = multi_to_dict(request.args)
# redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
exported_models = current_app.config.get('EXPORT_CSV_MODELS', [])
if not params and 'dataset' in exported_models:
return redirect(get_export_url('dataset'))
params['facets'] = False
datasets = search.iter(Dataset, **params)
adapter = csv.get_adapter(Dataset)
return csv.stream(adapter(datasets), 'datasets')
def datasets(topic):
kwargs = multi_to_dict(request.args)
kwargs.pop('topic', None)
topic_search = topic_search_for(topic,
DatasetSearch,
facets=True,
**kwargs)
return theme.render(
'topic/datasets.html',
topic=topic,
datasets=search.query(topic_search)
)
def reuses(topic):
kwargs = multi_to_dict(request.args)
kwargs.pop('topic', None)
topic_search = topic_search_for(topic,
ReuseSearch,
facets=True,
**kwargs)
return theme.render(
'topic/reuses.html',
topic=topic,
reuses=search.query(topic_search)
)
def display(topic):
kwargs = multi_to_dict(request.args)
kwargs.update(topic=topic)
datasets, reuses = multiquery(
TopicSearchQuery(DatasetSearch, **kwargs),
TopicSearchQuery(ReuseSearch, **kwargs),
)
return render('topic/display.html',
topic=topic,
datasets=datasets,
reuses=reuses,
)
def render_search():
params = multi_to_dict(request.args)
params['facets'] = True
datasets, organizations, reuses, users = search.multiquery(
search.SearchQuery(Dataset, **params),
search.SearchQuery(Organization, **params),
search.SearchQuery(Reuse, **params),
search.SearchQuery(User, **params),
)
return theme.render('search.html',
datasets=datasets,
organizations=organizations,
reuses=reuses,
users=users
)
def test_to_url_with_none(self):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.SearchQuery(FakeSearch, **kwargs)
with self.app.test_request_context('/an_url'):
url = search_query.to_url(tag=None, other='value', replace=True)
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
self.assertEqual(parsed_url.path, '/an_url')
self.assertEqual(multi_to_dict(qs), {
'q': 'test',
'other': 'value',
})
def test_to_url_with_override(self):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.search_for(FakeSearch, **kwargs)
with self.app.test_request_context('/an_url'):
url = search_query.to_url(tag='tag3', other='value')
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
self.assertEqual(parsed_url.path, '/an_url')
self.assert_dict_equal(multi_to_dict(qs), {
'q': 'test',
'tag': ['tag1', 'tag2', 'tag3'],
'other': 'value',
})
def test_to_url_with_specified_url(self, app):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.search_for(FakeSearch, **kwargs)
with app.test_request_context('/an_url'):
url = search_query.to_url('/another_url')
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
assert parsed_url.path == '/another_url'
assert_json_equal(multi_to_dict(qs), {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': '2',
})
def test_to_url_with_specified_url(self):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.SearchQuery(FakeSearch, **kwargs)
with self.app.test_request_context('/an_url'):
url = search_query.to_url('/another_url')
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
self.assertEqual(parsed_url.path, '/another_url')
self.assertEqual(multi_to_dict(qs), {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': '2',
})
def test_to_url_with_override_and_replace(self, app):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.search_for(FakeSearch, **kwargs)
with app.test_request_context('/an_url'):
url = search_query.to_url(tag='tag3', other='value', replace=True)
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
assert parsed_url.path == '/an_url'
assert_json_equal(multi_to_dict(qs), {
'q': 'test',
'tag': 'tag3',
'other': 'value',
})
def test_to_url_with_specified_url(self):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
}
search_query = search.SearchQuery(FakeSearch, **kwargs)
with self.app.test_request_context('/an_url'):
url = search_query.to_url('/another_url')
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
self.assertEqual(parsed_url.path, '/another_url')
self.assertEqual(multi_to_dict(qs), {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': '2',
})
def test_to_url(self):
kwargs = {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': 2,
'facets': True,
}
search_query = search.search_for(FakeSearch, **kwargs)
with self.app.test_request_context('/an_url'):
url = search_query.to_url()
parsed_url = url_parse(url)
qs = url_decode(parsed_url.query)
self.assertEqual(parsed_url.path, '/an_url')
self.assert_dict_equal(multi_to_dict(qs), {
'q': 'test',
'tag': ['tag1', 'tag2'],
'page': '2',
})
def compute_territory_dataset(territory, dataset, resource_id):
"""
Dynamically generate a CSV file about the territory from a national one.
The file targeted by the resource MUST be downloaded within the
`references` folder with the original name prior to call that view.
The GET paramaters `territory_attr` and `csv_column` are used to
determine which attribute MUST match the given column of the CSV.
"""
args = multi_to_dict(request.args)
if 'territory_attr' not in args or 'csv_column' not in args:
return abort(404)
if not hasattr(territory, args['territory_attr']):
return abort(400)
for resource in dataset.resources:
if resource.id == resource_id:
break
resource_path = references.path(resource.url.split('/')[-1])
match = getattr(territory, args['territory_attr']).encode('utf-8')
csvfile_out = StringIO.StringIO()
with open(resource_path, 'rb') as csvfile_in:
reader = csv.DictReader(csvfile_in, delimiter=str(';'))
writer = csv.DictWriter(csvfile_out, fieldnames=reader.fieldnames)
writer.writerow(dict(zip(writer.fieldnames, writer.fieldnames)))
for row in reader:
if row[args['csv_column']].encode('utf-8') == match:
writer.writerow(row)
csvfile_out.seek(0) # Back to 0 otherwise the file is served empty.
attachment_filename = '{territory_name}_{resource_name}.csv'.format(
territory_name=territory.name,
resource_name=resource.title.replace(' ', '_'))
return send_file(csvfile_out,
as_attachment=True,
attachment_filename=attachment_filename)
def compute_territory_dataset(territory, dataset, resource_id):
"""
Dynamically generate a CSV file about the territory from a national one.
The file targeted by the resource MUST be downloaded within the
`references` folder with the original name prior to call that view.
The GET paramaters `territory_attr` and `csv_column` are used to
determine which attribute MUST match the given column of the CSV.
"""
args = multi_to_dict(request.args)
if 'territory_attr' not in args or 'csv_column' not in args:
return abort(404)
if not hasattr(territory, args['territory_attr']):
return abort(400)
for resource in dataset.resources:
if resource.id == resource_id:
break
resource_path = references.path(resource.url.split('/')[-1])
match = getattr(territory, args['territory_attr']).encode('utf-8')
csvfile_out = StringIO.StringIO()
with open(resource_path, 'rb') as csvfile_in:
reader = csv.DictReader(csvfile_in, delimiter=str(';'))
writer = csv.DictWriter(csvfile_out, fieldnames=reader.fieldnames)
writer.writerow(dict(zip(writer.fieldnames, writer.fieldnames)))
for row in reader:
if row[args['csv_column']].encode('utf-8') == match:
writer.writerow(row)
csvfile_out.seek(0) # Back to 0 otherwise the file is served empty.
attachment_filename = '{territory_name}_{resource_name}.csv'.format(
territory_name=territory.name,
resource_name=resource.title.replace(' ', '_'))
return send_file(csvfile_out, as_attachment=True,
attachment_filename=attachment_filename)
def get_queryset(self):
return search.query(self.search_adapter, **multi_to_dict(request.args))
def get_queryset(self):
params = multi_to_dict(request.args)
params['facets'] = True
return search.query(self.search_adapter or self.model, **params)
def get(self):
'''List or search all organizations'''
search_parser.parse_args()
return search.query(OrganizationSearch, **multi_to_dict(request.args))
def get(self):
'''List or search all datasets'''
return search.query(DatasetSearch, **multi_to_dict(request.args))
def get(self):
'''List all users'''
search_parser.parse_args()
return search.query(UserSearch, **multi_to_dict(request.args))
def get(self):
search_parser.parse_args()
return search.query(ReuseSearch, **multi_to_dict(request.args))
def organizations_csv():
params = multi_to_dict(request.args)
params['facets'] = False
organizations = search.iter(Organization, **params)
return csv.stream(OrganizationCsvAdapter(organizations), 'organizations')
def datasets_csv():
params = multi_to_dict(request.args)
params['facets'] = False
datasets = search.iter(Dataset, **params)
adapter = csv.get_adapter(Dataset)
return csv.stream(adapter(datasets), 'datasets')
def get(self):
return search.query(ReuseSearch, **multi_to_dict(request.args))
def get(self):
'''List all users'''
search_parser.parse_args()
return search.query(UserSearch, **multi_to_dict(request.args))
def get(self):
'''List or search all datasets'''
return search.query(DatasetSearch, **multi_to_dict(request.args))
def get(self):
'''List or search all datasets'''
search_parser.parse_args()
return search.query(Dataset, **multi_to_dict(request.args))
def get(self):
return search.query(DatasetSearch, **multi_to_dict(request.args))
def resources_csv():
params = multi_to_dict(request.args)
params['facets'] = False
datasets = search.iter(Dataset, **params)
return csv.stream(ResourcesCsvAdapter(datasets), 'resources')
def get(self):
return search.query(ReuseSearch, **multi_to_dict(request.args))
def reuses_csv():
params = multi_to_dict(request.args)
params['facets'] = False
reuses = search.iter(Reuse, **params)
return csv.stream(ReuseCsvAdapter(reuses), 'reuses')
def get(self):
return search.query(DatasetSearch, **multi_to_dict(request.args))
def get(self):
'''List or search all datasets'''
search_parser.parse_args()
return search.query(Dataset, **multi_to_dict(request.args))
def organizations_csv():
params = multi_to_dict(request.args)
params['facets'] = False
organizations = search.iter(Organization, **params)
return csv.stream(OrganizationCsvAdapter(organizations), 'organizations')