def datasets_series_list(dataset): """Return all series for one dataset """ query = {'enable': True, 'slug': dataset} projection = {"_id": False, "values": False} dataset_doc = queries.col_datasets().find_one(query, projection) if not dataset_doc: abort(404) query = { 'provider_name': dataset_doc['provider_name'], 'dataset_code': dataset_doc["dataset_code"] } projection = {"_id": False, "values": False} query = queries.complex_queries_series(query) limit = request.args.get('limit', default=1000, type=int) docs = queries.col_series().find(query, projection) if limit: docs = docs.limit(limit) _docs = [doc for doc in docs] return json_tools.json_response(_docs)
def ajax_plot_series(slug): series = queries.col_series().find_one({"slug": slug}) if not series: abort(404) ds_projection = {"enable": True} dataset = queries.col_datasets().find_one( { "provider_name": series["provider_name"], "dataset_code": series["dataset_code"] }, ds_projection) if not dataset: abort(404) if dataset["enable"] is False: abort(307) meta = { "provider_name": series["provider_name"], "dataset_code": series["dataset_code"], "name": series["name"], "key": series["key"], "slug": series["slug"], } datas = [] #datas.append(("Dates", "Values")) for period, period_ts, value in datas_from_series(series): datas.append({ "period": period, "period_ts": str(period_ts), "value": value }) return json_tools.json_response(datas, meta)
def dataset_unit(dataset): query = {'enable': True, 'slug': dataset} projection = {"_id": False, "enable": False, "lock": False, "tags": False} doc = queries.col_datasets().find_one(query, projection) if not doc: abort(404) return json_tools.json_response(doc)
def datasets_series_list(dataset): """Return all series for one dataset """ query = {'enable': True, 'slug': dataset} projection = {"_id": False, "values": False} dataset_doc = queries.col_datasets().find_one(query, projection) if not dataset_doc: abort(404) query = { 'provider_name': dataset_doc['provider_name'], 'dataset_code': dataset_doc["dataset_code"] } projection = {"_id": False, "values": False} query = queries.complex_queries_series(query) docs = queries.col_series().find(query, projection) page = request.args.get('page', default=1, type=int) per_page = request.args.get('per_page', default=20, type=int) if per_page > 100: per_page = 100 pagination = queries.Pagination(docs, page, per_page) meta = { "page": pagination.page, "pages": pagination.pages, "per_page": pagination.per_page, "total": pagination.total, } _docs = [doc for doc in pagination.items] return json_tools.json_response(_docs, meta=meta)
def datasets_series_list(dataset): """Return all series for one dataset """ query = {'enable': True, 'slug': dataset} projection = {"_id": False, "values": False} dataset_doc = queries.col_datasets().find_one(query, projection) if not dataset_doc: abort(404) query = {'provider_name': dataset_doc['provider_name'], 'dataset_code': dataset_doc["dataset_code"]} projection = {"_id": False, "values": False} query = queries.complex_queries_series(query) docs = queries.col_series().find(query, projection) page = request.args.get('page', default=1, type=int) per_page = request.args.get('per_page', default=20, type=int) if per_page > 100: per_page = 100 pagination = queries.Pagination(docs, page, per_page) meta = { "page": pagination.page, "pages": pagination.pages, "per_page": pagination.per_page, "total": pagination.total, } _docs = [doc for doc in pagination.items] return json_tools.json_response(_docs, meta=meta)
def datasets_attributes_keys(dataset): query = {'enable': True, 'slug': dataset} projection = {"_id": False, "attribute_keys": True} doc = queries.col_datasets().find_one(query, projection) if not doc: abort(404) return json_tools.json_response(doc.get("attribute_keys"))
def datasets_codelists(dataset): query = {'enable': True, 'slug': dataset} projection = {"_id": False, "codelists": True} doc = queries.col_datasets().find_one(query, projection) if not doc: abort(404) return json_tools.json_response(doc["codelists"])
def stats_run_json(): query = {} provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] query["dataset_code"] = dataset["dataset_code"] elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] startDate = arrow.get(request.args.get('startDate')).floor('day').datetime endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime query["created"] = {"$gte": startDate, "$lte": endDate} limit = request.args.get('limit', default=100, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] """ for row in rows: row["view"] = url_for("views.dataset-by-code", provider_name=row["provider_name"], dataset_code=row["dataset_code"]) """ return json_tools.json_response(rows, {"total": count})
def ajax_datasets_list(provider): """ TODO: covered query provider_name + enable + dataset_code + name + slug ??? partialfilter sur enable = True """ provider_doc = queries.get_provider(provider) query = {'provider_name': provider_doc["name"]} projection = { "_id": False, "enable": True, "dataset_code": True, "name": True, "slug": True } is_meta = request.args.get('is_meta', type=int, default=0) == 1 if is_meta: projection["metadata"] = True docs = [ doc for doc in queries.col_datasets().find(query, projection) if doc["enable"] is True ] return json_tools.json_response(docs)
def ajax_plot_series(slug): series = queries.col_series().find_one({"slug": slug}) if not series: abort(404) ds_projection = {"enable": True} dataset = queries.col_datasets().find_one({"provider_name": series["provider_name"], "dataset_code": series["dataset_code"]}, ds_projection) if not dataset: abort(404) if dataset["enable"] is False: abort(307) meta = { "provider_name": series["provider_name"], "dataset_code": series["dataset_code"], "name": series["name"], "key": series["key"], "slug": series["slug"], } datas = [] #datas.append(("Dates", "Values")) for period, period_ts, value in datas_from_series(series): datas.append({"period": period, "period_ts": str(period_ts), "value": value}) return json_tools.json_response(datas, meta)
def datasets_last_update(): is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("datasets-last-update.html") query = { "$or": [{"count_inserts": {"$gt": 0}}, {"count_updates": {"$gt": 0}}] } startDate = arrow.utcnow().replace(days=-1).floor("second") query["created"] = {"$gte": startDate.datetime} limit = request.args.get('limit', default=0, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] for row in rows: slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]), word_boundary=False, save_order=True) row["view"] = url_for(".explorer_d", dataset=slug) return json_tools.json_response(rows, {"total": count})
def datasets_series_list(dataset): """Return all series for one dataset """ query = {'enable': True, 'slug': dataset} projection = {"_id": False, "values": False} dataset_doc = queries.col_datasets().find_one(query, projection) if not dataset_doc: abort(404) query = {'provider_name': dataset_doc['provider_name'], 'dataset_code': dataset_doc["dataset_code"]} projection = {"_id": False, "values": False} query = queries.complex_queries_series(query) limit = request.args.get('limit', default=1000, type=int) docs = queries.col_series().find(query, projection) if limit: docs= docs.limit(limit) _docs = [doc for doc in docs] return json_tools.json_response(_docs)
def ajax_providers_list(): query = {"enable": True} projection = {"_id": False, "slug": True, "name": True} docs = [ doc for doc in queries.col_providers().find(query, projection).sort( "name", ASCENDING) ] return json_tools.json_response(docs)
def providers_datasets_list(provider): provider_doc = queries.get_provider(provider) query = {'provider_name': provider_doc["name"]} projection = {"_id": False, "tags": False, "enable": False, "lock": False, "concepts": False, "codelists": False} #docs = [doc for doc in queries.col_datasets().find(query, projection)] docs = [doc for doc in queries.col_datasets().find(query, projection)] return json_tools.json_response(docs)
def datasets_attributes_list(dataset): query = {'enable': True, 'slug': dataset} projection = {"_id": False, "attribute_keys": True, "codelists": True} doc = queries.col_datasets().find_one(query, projection) if not doc: abort(404) attributes = OrderedDict() for key in doc.get("attribute_keys"): attributes[key] = doc["codelists"].get(key, {}) return json_tools.json_response(attributes)
def providers_unit(provider): projection = { "_id": False, "lock": False, "metadata": False, "enable": False, "version": False } provider_doc = queries.get_provider(provider, projection=projection) return json_tools.json_response(provider_doc)
def providers_list(): query = {"enable": True} projection = { "_id": False, "lock": False, "metadata": False, "enable": False, "version": False } docs = [doc for doc in queries.col_providers().find(query, projection)] return json_tools.json_response(docs)
def ajax_datasets_dimensions_keys(dataset): projection = {"_id": False, 'enable': True, "dimension_keys": True, "concepts": True} doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["concepts"]: dimensions.append({"value": key, "text": doc["concepts"][key] }) else: dimensions.append({"value": key, "text": key }) return json_tools.json_response(dimensions)
def dataset_unit_frequencies(dataset): query = {'enable': True, 'slug': dataset} projection = {"_id": False, "enable": False, "lock": False, "tags": False} doc = queries.col_datasets().find_one(query, projection) if not doc: abort(404) query = {"provider_name": doc["provider_name"], "dataset_code": doc["dataset_code"]} docs = queries.col_series().distinct("frequency", filter=query) return json_tools.json_response(docs)
def ajax_datasets_dimensions_all(dataset): projection = {"_id": False, "enable": True, "dimension_keys": True, "concepts": True, "codelists": True} doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["codelists"] and doc["codelists"][key]: dimensions.append({ "key": key, "name": doc["concepts"].get(key) or key, "codes": doc["codelists"][key], }) return json_tools.json_response(dimensions)
def providers_datasets_list(provider): provider_doc = queries.get_provider(provider) query = {'provider_name': provider_doc["name"]} projection = { "_id": False, "tags": False, "enable": False, "lock": False, "concepts": False, "codelists": False } #docs = [doc for doc in queries.col_datasets().find(query, projection)] docs = [doc for doc in queries.col_datasets().find(query, projection)] return json_tools.json_response(docs)
def dataset_unit_frequencies(dataset): query = {'enable': True, 'slug': dataset} projection = {"_id": False, "enable": False, "lock": False, "tags": False} doc = queries.col_datasets().find_one(query, projection) if not doc: abort(404) query = { "provider_name": doc["provider_name"], "dataset_code": doc["dataset_code"] } docs = queries.col_series().distinct("frequency", filter=query) return json_tools.json_response(docs)
def series_unit(series): query = {'slug': series} projection = {"_id": False} doc = queries.col_series().find_one(query, projection) if not doc: abort(404) query = {'enable': True, "provider_name": doc["provider_name"], 'dataset_code': doc["dataset_code"]} projection = {"_id": False, "enable": True} dataset_doc = queries.col_datasets().find_one(query, projection) if not dataset_doc: abort(404) return json_tools.json_response(doc)
def ajax_datasets_dimensions_keys(dataset): projection = { "_id": False, 'enable': True, "dimension_keys": True, "concepts": True } doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["concepts"]: dimensions.append({"value": key, "text": doc["concepts"][key]}) else: dimensions.append({"value": key, "text": key}) return json_tools.json_response(dimensions)
def series_unit(series): query = {'slug': series} projection = {"_id": False} doc = queries.col_series().find_one(query, projection) if not doc: abort(404) query = { 'enable': True, "provider_name": doc["provider_name"], 'dataset_code': doc["dataset_code"] } projection = {"_id": False, "enable": True} dataset_doc = queries.col_datasets().find_one(query, projection) if not dataset_doc: abort(404) return json_tools.json_response(doc)
def ajax_dataset_frequencies(dataset): projection = {"_id": False, "lock": False, "tags": False} doc = queries.get_dataset(dataset, projection) query = {"provider_name": doc["provider_name"], "dataset_code": doc["dataset_code"]} if "metadata" in doc and doc["metadata"].get("frequencies"): frequencies = doc["metadata"].get("frequencies") else: frequencies = queries.col_series().distinct("frequency", filter=query) freqs = [] for freq in frequencies: if freq in constants.FREQUENCIES_DICT: freqs.append({"value": freq, "text": constants.FREQUENCIES_DICT[freq]}) else: freqs.append({"value": freq, "text": freq}) return json_tools.json_response(freqs)
def ajax_datasets_list(provider): """ TODO: covered query provider_name + enable + dataset_code + name + slug ??? partialfilter sur enable = True """ provider_doc = queries.get_provider(provider) query = {'provider_name': provider_doc["name"]} projection = {"_id": False, "enable": True, "dataset_code": True, "name": True, "slug": True} is_meta = request.args.get('is_meta', type=int, default=0) == 1 if is_meta: projection["metadata"] = True docs = [doc for doc in queries.col_datasets().find(query, projection) if doc["enable"] is True] return json_tools.json_response(docs)
def ajax_datasets_dimensions_all(dataset): projection = { "_id": False, "enable": True, "dimension_keys": True, "concepts": True, "codelists": True } doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["codelists"] and doc["codelists"][key]: dimensions.append({ "key": key, "name": doc["concepts"].get(key) or key, "codes": doc["codelists"][key], }) return json_tools.json_response(dimensions)
def ajax_dataset_frequencies(dataset): projection = {"_id": False, "lock": False, "tags": False} doc = queries.get_dataset(dataset, projection) query = { "provider_name": doc["provider_name"], "dataset_code": doc["dataset_code"] } if "metadata" in doc and doc["metadata"].get("frequencies"): frequencies = doc["metadata"].get("frequencies") else: frequencies = queries.col_series().distinct("frequency", filter=query) freqs = [] for freq in frequencies: if freq in constants.FREQUENCIES_DICT: freqs.append({ "value": freq, "text": constants.FREQUENCIES_DICT[freq] }) else: freqs.append({"value": freq, "text": freq}) return json_tools.json_response(freqs)
def datasets_last_update(): is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("datasets-last-update.html") query = { "$or": [{ "count_inserts": { "$gt": 0 } }, { "count_updates": { "$gt": 0 } }] } startDate = arrow.utcnow().replace(days=-1).floor("second") query["created"] = {"$gte": startDate.datetime} limit = request.args.get('limit', default=0, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] for row in rows: slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]), word_boundary=False, save_order=True) row["view"] = url_for(".explorer_d", dataset=slug) return json_tools.json_response(rows, {"total": count})
def providers_list_keys(): docs = queries.col_providers().distinct("slug", filter={"enable": True}) return json_tools.json_response(docs)
def providers_list(): query = {"enable": True} projection = {"_id": False, "lock": False, "metadata": False, "enable": False, "version": False} docs = [doc for doc in queries.col_providers().find(query, projection)] return json_tools.json_response(docs)
def providers_unit(provider): projection = {"_id": False, "lock": False, "metadata": False, "enable": False, "version": False} provider_doc = queries.get_provider(provider, projection=projection) return json_tools.json_response(provider_doc)
def series_multi(series): query = {'slug': {"$in": series.split("+")}} projection = {"_id": False} docs = [doc for doc in queries.col_series().find(query, projection)] return json_tools.json_response(docs)
def ajax_explorer_datas(): limit = request.args.get('limit', default=100, type=int) if limit > 1000: limit = 1000 #7 669 115 octets pour 1000 series projection = { "_id": False, "dimensions": False, "attributes": False, "values.revisions": False, "notes": False, "tags": False } query = OrderedDict() provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') #series_slug = request.args.get('series') search = request.args.get('search') is_eurostat = False if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] #query["dataset_code"] = dataset["dataset_code"] if dataset["provider_name"] == "EUROSTAT": is_eurostat = True elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] if provider["name"] == "EUROSTAT": is_eurostat = True if search: query["$text"] = {"$search": search.strip()} projection['score'] = {'$meta': 'textScore'} disabled_datasets = [] if dataset_slug: query["dataset_code"] = dataset["dataset_code"] else: ds_enabled_query = {"enable": False} if "provider_name" in query: ds_enabled_query["provider_name"] = query["provider_name"] disabled_datasets = [doc["dataset_code"] for doc in queries.col_datasets().find(ds_enabled_query, {"dataset_code": True})] query = complex_queries_series(query) cursor = queries.col_series().find(dict(query), projection) if search: cursor = cursor.sort([('score', {'$meta': 'textScore'})]) #else: # query = {"slug": series_slug} # cursor = queries.col_series().find(dict(query), projection) if limit: cursor = cursor.limit(limit) if is_eurostat: count = 0 else: count = cursor.count() series_list = [doc for doc in cursor] rows = [] for s in series_list: if disabled_datasets and s["dataset_code"] in disabled_datasets: continue if not "version" in s: s["version"] = 0 s['start_date'] = s["values"][0]["period"] s['end_date'] = s["values"][-1]["period"] values = [{"period": v["period"], "value": v["value"]} for v in s['values']] del s["values"] s["values"] = values s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) #s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug']) #TODO: s['url_dataset'] = url_for('.dataset', id=s['_id']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] rows.append(s) return json_tools.json_response(rows, {"total": count})
def providers_unit(provider): provider_doc = queries.get_provider(provider) return json_tools.json_response(provider_doc)
def datasets_list_keys(provider): provider_doc = queries.get_provider(provider) query = {'provider_name': provider_doc["name"]} docs = queries.col_datasets().distinct("slug", filter=query) return json_tools.json_response(docs)
def ajax_explorer_datas(): limit = request.args.get('limit', default=100, type=int) if limit > 1000: limit = 1000 #7 669 115 octets pour 1000 series projection = { "_id": False, "dimensions": False, "attributes": False, "values.revisions": False, "notes": False, "tags": False } query = OrderedDict() provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') #series_slug = request.args.get('series') search = request.args.get('search') is_eurostat = False if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] #query["dataset_code"] = dataset["dataset_code"] if dataset["provider_name"] == "EUROSTAT": is_eurostat = True elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] if provider["name"] == "EUROSTAT": is_eurostat = True if search: query["$text"] = {"$search": search.strip()} projection['score'] = {'$meta': 'textScore'} disabled_datasets = [] if dataset_slug: query["dataset_code"] = dataset["dataset_code"] else: ds_enabled_query = {"enable": False} if "provider_name" in query: ds_enabled_query["provider_name"] = query["provider_name"] disabled_datasets = [ doc["dataset_code"] for doc in queries.col_datasets().find( ds_enabled_query, {"dataset_code": True}) ] query = complex_queries_series(query) cursor = queries.col_series().find(dict(query), projection) if search: cursor = cursor.sort([('score', {'$meta': 'textScore'})]) #else: # query = {"slug": series_slug} # cursor = queries.col_series().find(dict(query), projection) if limit: cursor = cursor.limit(limit) if is_eurostat: count = 0 else: count = cursor.count() series_list = [doc for doc in cursor] rows = [] for s in series_list: if disabled_datasets and s["dataset_code"] in disabled_datasets: continue if not "version" in s: s["version"] = 0 s['start_date'] = s["values"][0]["period"] s['end_date'] = s["values"][-1]["period"] values = [{ "period": v["period"], "value": v["value"] } for v in s['values']] del s["values"] s["values"] = values s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) #s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug']) #TODO: s['url_dataset'] = url_for('.dataset', id=s['_id']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] rows.append(s) return json_tools.json_response(rows, {"total": count})
def ajax_providers_list(): query = {"enable": True} projection = {"_id": False, "slug": True, "name": True} docs = [doc for doc in queries.col_providers().find(query, projection).sort("name", ASCENDING)] return json_tools.json_response(docs)