def dataset_with_slug(slug):
dataset = queries.get_dataset(slug, {"_id": False})
is_modal = request.args.get('modal', default=0, type=int)
provider = queries.col_providers().find_one(
{"name": dataset['provider_name']}, {"metadata": False})
#count_series = queries.col_series().count({"provider_name": dataset['provider_name'],
# "dataset_code": dataset['dataset_code']})
url_provider = url_for('.explorer_p', provider=provider["slug"])
url_dataset = url_for('.explorer_d', dataset=dataset["slug"])
url_dataset_direct = url_for('.dataset-by-slug',
slug=dataset["slug"],
_external=True)
return render_template(
"dataset-unit-modal.html",
is_modal=is_modal,
url_provider=url_provider,
url_dataset=url_dataset,
url_dataset_direct=url_dataset_direct,
dataset=dataset,
provider=provider,
#count=count_series
)
def explorer_view(provider=None, dataset=None, series=None):
"""
http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer/insee
http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer
http://127.0.0.1:8081/views
"""
if not dataset and not provider:
provider = current_app.config.get('DEFAULT_PROVIDER', None)
dataset = current_app.config.get('DEFAULT_DATASET', None)
if dataset:
doc = queries.get_dataset(dataset)
#provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True})
provider = doc["provider_name"].lower()
elif provider:
provider_doc = queries.get_provider(provider, {"slug": True, "name": True, "enable": True})
dataset_doc = queries.col_datasets().find_one({"provider_name": provider_doc["name"],
"enable": True},
{"slug": True})
dataset = dataset_doc["slug"]
ctx = {
"selectedProvider": provider,
"selectedDataset": dataset,
}
return render_template("explorer.html", **ctx)
def stats_run_json():
query = {}
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
query["dataset_code"] = dataset["dataset_code"]
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
startDate = arrow.get(request.args.get('startDate')).floor('day').datetime
endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime
query["created"] = {"$gte": startDate, "$lte": endDate}
limit = request.args.get('limit', default=100, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
"""
for row in rows:
row["view"] = url_for("views.dataset-by-code",
provider_name=row["provider_name"],
dataset_code=row["dataset_code"])
"""
return json_tools.json_response(rows, {"total": count})
def stats_run_json():
query = {}
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
query["dataset_code"] = dataset["dataset_code"]
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
startDate = arrow.get(request.args.get('startDate')).floor('day').datetime
endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime
query["created"] = {"$gte": startDate, "$lte": endDate}
limit = request.args.get('limit', default=100, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
"""
for row in rows:
row["view"] = url_for("views.dataset-by-code",
provider_name=row["provider_name"],
dataset_code=row["dataset_code"])
"""
return json_tools.json_response(rows, {"total": count})
def ajax_datasets_dimensions_keys(dataset):
projection = {"_id": False, 'enable': True, "dimension_keys": True, "concepts": True}
doc = queries.get_dataset(dataset, projection)
dimensions = []
for key in doc["dimension_keys"]:
if key in doc["concepts"]:
dimensions.append({"value": key, "text": doc["concepts"][key] })
else:
dimensions.append({"value": key, "text": key })
return json_tools.json_response(dimensions)
def ajax_datasets_dimensions_all(dataset):
projection = {"_id": False, "enable": True, "dimension_keys": True, "concepts": True, "codelists": True}
doc = queries.get_dataset(dataset, projection)
dimensions = []
for key in doc["dimension_keys"]:
if key in doc["codelists"] and doc["codelists"][key]:
dimensions.append({
"key": key,
"name": doc["concepts"].get(key) or key,
"codes": doc["codelists"][key],
})
return json_tools.json_response(dimensions)
def ajax_datasets_dimensions_keys(dataset):
projection = {
"_id": False,
'enable': True,
"dimension_keys": True,
"concepts": True
}
doc = queries.get_dataset(dataset, projection)
dimensions = []
for key in doc["dimension_keys"]:
if key in doc["concepts"]:
dimensions.append({"value": key, "text": doc["concepts"][key]})
else:
dimensions.append({"value": key, "text": key})
return json_tools.json_response(dimensions)
def ajax_dataset_frequencies(dataset):
projection = {"_id": False, "lock": False, "tags": False}
doc = queries.get_dataset(dataset, projection)
query = {"provider_name": doc["provider_name"],
"dataset_code": doc["dataset_code"]}
if "metadata" in doc and doc["metadata"].get("frequencies"):
frequencies = doc["metadata"].get("frequencies")
else:
frequencies = queries.col_series().distinct("frequency", filter=query)
freqs = []
for freq in frequencies:
if freq in constants.FREQUENCIES_DICT:
freqs.append({"value": freq, "text": constants.FREQUENCIES_DICT[freq]})
else:
freqs.append({"value": freq, "text": freq})
return json_tools.json_response(freqs)
def ajax_datasets_dimensions_all(dataset):
projection = {
"_id": False,
"enable": True,
"dimension_keys": True,
"concepts": True,
"codelists": True
}
doc = queries.get_dataset(dataset, projection)
dimensions = []
for key in doc["dimension_keys"]:
if key in doc["codelists"] and doc["codelists"][key]:
dimensions.append({
"key": key,
"name": doc["concepts"].get(key) or key,
"codes": doc["codelists"][key],
})
return json_tools.json_response(dimensions)
def ajax_dataset_frequencies(dataset):
projection = {"_id": False, "lock": False, "tags": False}
doc = queries.get_dataset(dataset, projection)
query = {
"provider_name": doc["provider_name"],
"dataset_code": doc["dataset_code"]
}
if "metadata" in doc and doc["metadata"].get("frequencies"):
frequencies = doc["metadata"].get("frequencies")
else:
frequencies = queries.col_series().distinct("frequency", filter=query)
freqs = []
for freq in frequencies:
if freq in constants.FREQUENCIES_DICT:
freqs.append({
"value": freq,
"text": constants.FREQUENCIES_DICT[freq]
})
else:
freqs.append({"value": freq, "text": freq})
return json_tools.json_response(freqs)
def explorer_view(provider=None, dataset=None, series=None):
"""
http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer/insee
http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88
http://127.0.0.1:8081/views/explorer
http://127.0.0.1:8081/views
"""
if not dataset and not provider:
provider = current_app.config.get('DEFAULT_PROVIDER', None)
dataset = current_app.config.get('DEFAULT_DATASET', None)
if dataset:
doc = queries.get_dataset(dataset)
# provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True})
provider = doc["provider_name"].lower()
elif provider:
provider_doc = queries.get_provider(provider, {
"slug": True,
"name": True,
"enable": True
})
dataset_doc = queries.col_datasets().find_one(
{
"provider_name": provider_doc["name"],
"enable": True
},
{"slug": True},
)
dataset = dataset_doc["slug"]
ctx = {
"selectedProvider": provider,
"selectedDataset": dataset,
}
return render_template("explorer.html", **ctx)
def dataset_with_slug(slug):
dataset = queries.get_dataset(slug, {"_id": False})
is_modal = request.args.get('modal', default=0, type=int)
provider = queries.col_providers().find_one({"name": dataset['provider_name']},
{"metadata": False})
#count_series = queries.col_series().count({"provider_name": dataset['provider_name'],
# "dataset_code": dataset['dataset_code']})
url_provider = url_for('.explorer_p', provider=provider["slug"])
url_dataset = url_for('.explorer_d', dataset=dataset["slug"])
url_dataset_direct = url_for('.dataset-by-slug', slug=dataset["slug"], _external=True)
return render_template("dataset-unit-modal.html",
is_modal=is_modal,
url_provider=url_provider,
url_dataset=url_dataset,
url_dataset_direct=url_dataset_direct,
dataset=dataset,
provider=provider,
#count=count_series
)
def ajax_explorer_datas():
limit = request.args.get('limit', default=100, type=int)
if limit > 1000:
limit = 1000
#7 669 115 octets pour 1000 series
projection = {
"_id": False,
"dimensions": False,
"attributes": False,
"values.revisions": False,
"notes": False,
"tags": False
}
query = OrderedDict()
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
#series_slug = request.args.get('series')
search = request.args.get('search')
is_eurostat = False
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
#query["dataset_code"] = dataset["dataset_code"]
if dataset["provider_name"] == "EUROSTAT":
is_eurostat = True
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
if provider["name"] == "EUROSTAT":
is_eurostat = True
if search:
query["$text"] = {"$search": search.strip()}
projection['score'] = {'$meta': 'textScore'}
disabled_datasets = []
if dataset_slug:
query["dataset_code"] = dataset["dataset_code"]
else:
ds_enabled_query = {"enable": False}
if "provider_name" in query:
ds_enabled_query["provider_name"] = query["provider_name"]
disabled_datasets = [
doc["dataset_code"] for doc in queries.col_datasets().find(
ds_enabled_query, {"dataset_code": True})
]
query = complex_queries_series(query)
cursor = queries.col_series().find(dict(query), projection)
if search:
cursor = cursor.sort([('score', {'$meta': 'textScore'})])
#else:
# query = {"slug": series_slug}
# cursor = queries.col_series().find(dict(query), projection)
if limit:
cursor = cursor.limit(limit)
if is_eurostat:
count = 0
else:
count = cursor.count()
series_list = [doc for doc in cursor]
rows = []
for s in series_list:
if disabled_datasets and s["dataset_code"] in disabled_datasets:
continue
if not "version" in s:
s["version"] = 0
s['start_date'] = s["values"][0]["period"]
s['end_date'] = s["values"][-1]["period"]
values = [{
"period": v["period"],
"value": v["value"]
} for v in s['values']]
del s["values"]
s["values"] = values
s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1)
dataset_slug = slugify("%s-%s" %
(s["provider_name"], s["dataset_code"]),
word_boundary=False,
save_order=True)
s['view_dataset'] = url_for('.dataset-by-slug',
slug=dataset_slug,
modal=1)
#s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True)
s['dataset_slug'] = dataset_slug
s['export_csv'] = url_for('.export-series-csv', slug=s['slug'])
s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug'])
s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug'])
#TODO: s['url_dataset'] = url_for('.dataset', id=s['_id'])
s['frequency_txt'] = s['frequency']
if s['frequency'] in constants.FREQUENCIES_DICT:
s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']]
rows.append(s)
return json_tools.json_response(rows, {"total": count})
def ajax_explorer_datas():
limit = request.args.get('limit', default=100, type=int)
if limit > 1000:
limit = 1000
#7 669 115 octets pour 1000 series
projection = {
"_id": False,
"dimensions": False,
"attributes": False,
"values.revisions": False,
"notes": False,
"tags": False
}
query = OrderedDict()
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
#series_slug = request.args.get('series')
search = request.args.get('search')
is_eurostat = False
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
#query["dataset_code"] = dataset["dataset_code"]
if dataset["provider_name"] == "EUROSTAT":
is_eurostat = True
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
if provider["name"] == "EUROSTAT":
is_eurostat = True
if search:
query["$text"] = {"$search": search.strip()}
projection['score'] = {'$meta': 'textScore'}
disabled_datasets = []
if dataset_slug:
query["dataset_code"] = dataset["dataset_code"]
else:
ds_enabled_query = {"enable": False}
if "provider_name" in query:
ds_enabled_query["provider_name"] = query["provider_name"]
disabled_datasets = [doc["dataset_code"] for doc in queries.col_datasets().find(ds_enabled_query,
{"dataset_code": True})]
query = complex_queries_series(query)
cursor = queries.col_series().find(dict(query), projection)
if search:
cursor = cursor.sort([('score', {'$meta': 'textScore'})])
#else:
# query = {"slug": series_slug}
# cursor = queries.col_series().find(dict(query), projection)
if limit:
cursor = cursor.limit(limit)
if is_eurostat:
count = 0
else:
count = cursor.count()
series_list = [doc for doc in cursor]
rows = []
for s in series_list:
if disabled_datasets and s["dataset_code"] in disabled_datasets:
continue
if not "version" in s:
s["version"] = 0
s['start_date'] = s["values"][0]["period"]
s['end_date'] = s["values"][-1]["period"]
values = [{"period": v["period"], "value": v["value"]} for v in s['values']]
del s["values"]
s["values"] = values
s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1)
dataset_slug = slugify("%s-%s" % (s["provider_name"],
s["dataset_code"]),
word_boundary=False, save_order=True)
s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1)
#s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True)
s['dataset_slug'] = dataset_slug
s['export_csv'] = url_for('.export-series-csv', slug=s['slug'])
s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug'])
s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug'])
#TODO: s['url_dataset'] = url_for('.dataset', id=s['_id'])
s['frequency_txt'] = s['frequency']
if s['frequency'] in constants.FREQUENCIES_DICT:
s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']]
rows.append(s)
return json_tools.json_response(rows, {"total": count})