def explorer_view(provider=None, dataset=None, series=None):
"""
http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer/insee
http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer
http://127.0.0.1:8081/views
"""
if not dataset and not provider:
provider = current_app.config.get('DEFAULT_PROVIDER', None)
dataset = current_app.config.get('DEFAULT_DATASET', None)
if dataset:
doc = queries.get_dataset(dataset)
#provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True})
provider = doc["provider_name"].lower()
elif provider:
provider_doc = queries.get_provider(provider, {"slug": True, "name": True, "enable": True})
dataset_doc = queries.col_datasets().find_one({"provider_name": provider_doc["name"],
"enable": True},
{"slug": True})
dataset = dataset_doc["slug"]
ctx = {
"selectedProvider": provider,
"selectedDataset": dataset,
}
return render_template("explorer.html", **ctx)
def ajax_datasets_list(provider):
"""
TODO: covered query
provider_name + enable + dataset_code + name + slug ???
partialfilter sur enable = True
"""
provider_doc = queries.get_provider(provider)
query = {'provider_name': provider_doc["name"]}
projection = {
"_id": False,
"enable": True,
"dataset_code": True,
"name": True,
"slug": True
}
is_meta = request.args.get('is_meta', type=int, default=0) == 1
if is_meta:
projection["metadata"] = True
docs = [
doc for doc in queries.col_datasets().find(query, projection)
if doc["enable"] is True
]
return json_tools.json_response(docs)
def stats_run_json():
query = {}
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
query["dataset_code"] = dataset["dataset_code"]
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
startDate = arrow.get(request.args.get('startDate')).floor('day').datetime
endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime
query["created"] = {"$gte": startDate, "$lte": endDate}
limit = request.args.get('limit', default=100, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
"""
for row in rows:
row["view"] = url_for("views.dataset-by-code",
provider_name=row["provider_name"],
dataset_code=row["dataset_code"])
"""
return json_tools.json_response(rows, {"total": count})
def stats_run_json():
query = {}
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
query["dataset_code"] = dataset["dataset_code"]
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
startDate = arrow.get(request.args.get('startDate')).floor('day').datetime
endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime
query["created"] = {"$gte": startDate, "$lte": endDate}
limit = request.args.get('limit', default=100, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
"""
for row in rows:
row["view"] = url_for("views.dataset-by-code",
provider_name=row["provider_name"],
dataset_code=row["dataset_code"])
"""
return json_tools.json_response(rows, {"total": count})
def ajax_tree_view(provider=None):
provider = provider or request.args.get('provider')
if not provider:
abort(404, "provider is required")
_provider = queries.get_provider(provider)
provider_name = _provider["name"]
query = {"provider_name": provider_name, "enable": True}
cursor = queries.col_categories().find(query, {"_id": False})
cursor = cursor.sort([("position", 1), ("category_code", 1)])
categories = OrderedDict([(doc["category_code"], doc) for doc in cursor])
ds_codes = []
for_remove = []
for cat in categories.values():
if cat.get("parent"):
parent = categories[cat.get("parent")]
if not "children" in parent:
parent["children"] = []
parent["children"].append(cat)
for_remove.append(cat["category_code"])
if cat.get("datasets"):
for ds in cat.get("datasets"):
ds_codes.append(ds["dataset_code"])
for r in for_remove:
categories.pop(r)
ds_query = {
'provider_name': provider_name,
"enable": True,
"dataset_code": {
"$in": list(set(ds_codes))
}
}
ds_projection = {"_id": True, "dataset_code": True, "slug": True}
cursor = queries.col_datasets().find(ds_query, ds_projection)
dataset_codes = {}
for doc in cursor:
dataset_codes[doc['dataset_code']] = {
"slug": doc['slug'],
"url": url_for('.dataset-by-slug', slug=doc["slug"])
}
context = dict(provider=_provider,
categories=categories,
dataset_codes=dataset_codes)
data = render_template("datatree_ajax.html", **context)
response = current_app.response_class(data,
mimetype='application/javascript')
response.status_code = 200
response.make_conditional(request)
return response
def ajax_tree_view(provider=None):
provider = provider or request.args.get('provider')
if not provider:
abort(404, "provider is required")
_provider = queries.get_provider(provider)
provider_name = _provider["name"]
query = {"provider_name": provider_name,
"enable": True}
cursor = queries.col_categories().find(query, {"_id": False})
cursor = cursor.sort([("position", 1), ("category_code", 1)])
categories = OrderedDict([(doc["category_code"], doc ) for doc in cursor])
ds_codes = []
for_remove = []
for cat in categories.values():
if cat.get("parent"):
parent = categories[cat.get("parent")]
if not "children" in parent:
parent["children"] = []
parent["children"].append(cat)
for_remove.append(cat["category_code"])
if cat.get("datasets"):
for ds in cat.get("datasets"):
ds_codes.append(ds["dataset_code"])
for r in for_remove:
categories.pop(r)
ds_query = {'provider_name': provider_name,
"enable": True,
"dataset_code": {"$in": list(set(ds_codes))}}
ds_projection = {"_id": True, "dataset_code": True, "slug": True}
cursor = queries.col_datasets().find(ds_query, ds_projection)
dataset_codes = {}
for doc in cursor:
dataset_codes[doc['dataset_code']] = {
"slug": doc['slug'],
"url": url_for('.dataset-by-slug', slug=doc["slug"])
}
context = dict(
provider=_provider,
categories=categories,
dataset_codes=dataset_codes
)
data = render_template("datatree_ajax.html", **context)
response = current_app.response_class(data,
mimetype='application/javascript')
response.status_code = 200
response.make_conditional(request)
return response
def ajax_datasets_list(provider):
"""
TODO: covered query
provider_name + enable + dataset_code + name + slug ???
partialfilter sur enable = True
"""
provider_doc = queries.get_provider(provider)
query = {'provider_name': provider_doc["name"]}
projection = {"_id": False, "enable": True,
"dataset_code": True, "name": True, "slug": True}
is_meta = request.args.get('is_meta', type=int, default=0) == 1
if is_meta:
projection["metadata"] = True
docs = [doc for doc in queries.col_datasets().find(query, projection) if doc["enable"] is True]
return json_tools.json_response(docs)
def explorer_view(provider=None, dataset=None, series=None):
"""
http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer/insee
http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer
http://127.0.0.1:8081/views
"""
if not dataset and not provider:
provider = current_app.config.get('DEFAULT_PROVIDER', None)
dataset = current_app.config.get('DEFAULT_DATASET', None)
if dataset:
doc = queries.get_dataset(dataset)
# provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True})
provider = doc["provider_name"].lower()
elif provider:
provider_doc = queries.get_provider(provider, {
"slug": True,
"name": True,
"enable": True
})
dataset_doc = queries.col_datasets().find_one(
{
"provider_name": provider_doc["name"],
"enable": True
},
{"slug": True},
)
dataset = dataset_doc["slug"]
ctx = {
"selectedProvider": provider,
"selectedDataset": dataset,
}
return render_template("explorer.html", **ctx)
def ajax_explorer_datas():
limit = request.args.get('limit', default=100, type=int)
if limit > 1000:
limit = 1000
#7 669 115 octets pour 1000 series
projection = {
"_id": False,
"dimensions": False,
"attributes": False,
"values.revisions": False,
"notes": False,
"tags": False
}
query = OrderedDict()
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
#series_slug = request.args.get('series')
search = request.args.get('search')
is_eurostat = False
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
#query["dataset_code"] = dataset["dataset_code"]
if dataset["provider_name"] == "EUROSTAT":
is_eurostat = True
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
if provider["name"] == "EUROSTAT":
is_eurostat = True
if search:
query["$text"] = {"$search": search.strip()}
projection['score'] = {'$meta': 'textScore'}
disabled_datasets = []
if dataset_slug:
query["dataset_code"] = dataset["dataset_code"]
else:
ds_enabled_query = {"enable": False}
if "provider_name" in query:
ds_enabled_query["provider_name"] = query["provider_name"]
disabled_datasets = [
doc["dataset_code"] for doc in queries.col_datasets().find(
ds_enabled_query, {"dataset_code": True})
]
query = complex_queries_series(query)
cursor = queries.col_series().find(dict(query), projection)
if search:
cursor = cursor.sort([('score', {'$meta': 'textScore'})])
#else:
# query = {"slug": series_slug}
# cursor = queries.col_series().find(dict(query), projection)
if limit:
cursor = cursor.limit(limit)
if is_eurostat:
count = 0
else:
count = cursor.count()
series_list = [doc for doc in cursor]
rows = []
for s in series_list:
if disabled_datasets and s["dataset_code"] in disabled_datasets:
continue
if not "version" in s:
s["version"] = 0
s['start_date'] = s["values"][0]["period"]
s['end_date'] = s["values"][-1]["period"]
values = [{
"period": v["period"],
"value": v["value"]
} for v in s['values']]
del s["values"]
s["values"] = values
s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1)
dataset_slug = slugify("%s-%s" %
(s["provider_name"], s["dataset_code"]),
word_boundary=False,
save_order=True)
s['view_dataset'] = url_for('.dataset-by-slug',
slug=dataset_slug,
modal=1)
#s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True)
s['dataset_slug'] = dataset_slug
s['export_csv'] = url_for('.export-series-csv', slug=s['slug'])
s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug'])
s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug'])
#TODO: s['url_dataset'] = url_for('.dataset', id=s['_id'])
s['frequency_txt'] = s['frequency']
if s['frequency'] in constants.FREQUENCIES_DICT:
s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']]
rows.append(s)
return json_tools.json_response(rows, {"total": count})
def ajax_explorer_datas():
limit = request.args.get('limit', default=100, type=int)
if limit > 1000:
limit = 1000
#7 669 115 octets pour 1000 series
projection = {
"_id": False,
"dimensions": False,
"attributes": False,
"values.revisions": False,
"notes": False,
"tags": False
}
query = OrderedDict()
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
#series_slug = request.args.get('series')
search = request.args.get('search')
is_eurostat = False
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
#query["dataset_code"] = dataset["dataset_code"]
if dataset["provider_name"] == "EUROSTAT":
is_eurostat = True
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
if provider["name"] == "EUROSTAT":
is_eurostat = True
if search:
query["$text"] = {"$search": search.strip()}
projection['score'] = {'$meta': 'textScore'}
disabled_datasets = []
if dataset_slug:
query["dataset_code"] = dataset["dataset_code"]
else:
ds_enabled_query = {"enable": False}
if "provider_name" in query:
ds_enabled_query["provider_name"] = query["provider_name"]
disabled_datasets = [doc["dataset_code"] for doc in queries.col_datasets().find(ds_enabled_query,
{"dataset_code": True})]
query = complex_queries_series(query)
cursor = queries.col_series().find(dict(query), projection)
if search:
cursor = cursor.sort([('score', {'$meta': 'textScore'})])
#else:
# query = {"slug": series_slug}
# cursor = queries.col_series().find(dict(query), projection)
if limit:
cursor = cursor.limit(limit)
if is_eurostat:
count = 0
else:
count = cursor.count()
series_list = [doc for doc in cursor]
rows = []
for s in series_list:
if disabled_datasets and s["dataset_code"] in disabled_datasets:
continue
if not "version" in s:
s["version"] = 0
s['start_date'] = s["values"][0]["period"]
s['end_date'] = s["values"][-1]["period"]
values = [{"period": v["period"], "value": v["value"]} for v in s['values']]
del s["values"]
s["values"] = values
s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1)
dataset_slug = slugify("%s-%s" % (s["provider_name"],
s["dataset_code"]),
word_boundary=False, save_order=True)
s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1)
#s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True)
s['dataset_slug'] = dataset_slug
s['export_csv'] = url_for('.export-series-csv', slug=s['slug'])
s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug'])
s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug'])
#TODO: s['url_dataset'] = url_for('.dataset', id=s['_id'])
s['frequency_txt'] = s['frequency']
if s['frequency'] in constants.FREQUENCIES_DICT:
s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']]
rows.append(s)
return json_tools.json_response(rows, {"total": count})
