def dataset_with_slug(slug): dataset = queries.get_dataset(slug, {"_id": False}) is_modal = request.args.get('modal', default=0, type=int) provider = queries.col_providers().find_one( {"name": dataset['provider_name']}, {"metadata": False}) #count_series = queries.col_series().count({"provider_name": dataset['provider_name'], # "dataset_code": dataset['dataset_code']}) url_provider = url_for('.explorer_p', provider=provider["slug"]) url_dataset = url_for('.explorer_d', dataset=dataset["slug"]) url_dataset_direct = url_for('.dataset-by-slug', slug=dataset["slug"], _external=True) return render_template( "dataset-unit-modal.html", is_modal=is_modal, url_provider=url_provider, url_dataset=url_dataset, url_dataset_direct=url_dataset_direct, dataset=dataset, provider=provider, #count=count_series )
def explorer_view(provider=None, dataset=None, series=None): """ http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer/insee http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer http://127.0.0.1:8081/views """ if not dataset and not provider: provider = current_app.config.get('DEFAULT_PROVIDER', None) dataset = current_app.config.get('DEFAULT_DATASET', None) if dataset: doc = queries.get_dataset(dataset) #provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True}) provider = doc["provider_name"].lower() elif provider: provider_doc = queries.get_provider(provider, {"slug": True, "name": True, "enable": True}) dataset_doc = queries.col_datasets().find_one({"provider_name": provider_doc["name"], "enable": True}, {"slug": True}) dataset = dataset_doc["slug"] ctx = { "selectedProvider": provider, "selectedDataset": dataset, } return render_template("explorer.html", **ctx)
def stats_run_json(): query = {} provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] query["dataset_code"] = dataset["dataset_code"] elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] startDate = arrow.get(request.args.get('startDate')).floor('day').datetime endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime query["created"] = {"$gte": startDate, "$lte": endDate} limit = request.args.get('limit', default=100, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] """ for row in rows: row["view"] = url_for("views.dataset-by-code", provider_name=row["provider_name"], dataset_code=row["dataset_code"]) """ return json_tools.json_response(rows, {"total": count})
def ajax_datasets_dimensions_keys(dataset): projection = {"_id": False, 'enable': True, "dimension_keys": True, "concepts": True} doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["concepts"]: dimensions.append({"value": key, "text": doc["concepts"][key] }) else: dimensions.append({"value": key, "text": key }) return json_tools.json_response(dimensions)
def ajax_datasets_dimensions_all(dataset): projection = {"_id": False, "enable": True, "dimension_keys": True, "concepts": True, "codelists": True} doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["codelists"] and doc["codelists"][key]: dimensions.append({ "key": key, "name": doc["concepts"].get(key) or key, "codes": doc["codelists"][key], }) return json_tools.json_response(dimensions)
def ajax_datasets_dimensions_keys(dataset): projection = { "_id": False, 'enable': True, "dimension_keys": True, "concepts": True } doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["concepts"]: dimensions.append({"value": key, "text": doc["concepts"][key]}) else: dimensions.append({"value": key, "text": key}) return json_tools.json_response(dimensions)
def ajax_dataset_frequencies(dataset): projection = {"_id": False, "lock": False, "tags": False} doc = queries.get_dataset(dataset, projection) query = {"provider_name": doc["provider_name"], "dataset_code": doc["dataset_code"]} if "metadata" in doc and doc["metadata"].get("frequencies"): frequencies = doc["metadata"].get("frequencies") else: frequencies = queries.col_series().distinct("frequency", filter=query) freqs = [] for freq in frequencies: if freq in constants.FREQUENCIES_DICT: freqs.append({"value": freq, "text": constants.FREQUENCIES_DICT[freq]}) else: freqs.append({"value": freq, "text": freq}) return json_tools.json_response(freqs)
def ajax_datasets_dimensions_all(dataset): projection = { "_id": False, "enable": True, "dimension_keys": True, "concepts": True, "codelists": True } doc = queries.get_dataset(dataset, projection) dimensions = [] for key in doc["dimension_keys"]: if key in doc["codelists"] and doc["codelists"][key]: dimensions.append({ "key": key, "name": doc["concepts"].get(key) or key, "codes": doc["codelists"][key], }) return json_tools.json_response(dimensions)
def ajax_dataset_frequencies(dataset): projection = {"_id": False, "lock": False, "tags": False} doc = queries.get_dataset(dataset, projection) query = { "provider_name": doc["provider_name"], "dataset_code": doc["dataset_code"] } if "metadata" in doc and doc["metadata"].get("frequencies"): frequencies = doc["metadata"].get("frequencies") else: frequencies = queries.col_series().distinct("frequency", filter=query) freqs = [] for freq in frequencies: if freq in constants.FREQUENCIES_DICT: freqs.append({ "value": freq, "text": constants.FREQUENCIES_DICT[freq] }) else: freqs.append({"value": freq, "text": freq}) return json_tools.json_response(freqs)
def explorer_view(provider=None, dataset=None, series=None): """ http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer/insee http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer http://127.0.0.1:8081/views """ if not dataset and not provider: provider = current_app.config.get('DEFAULT_PROVIDER', None) dataset = current_app.config.get('DEFAULT_DATASET', None) if dataset: doc = queries.get_dataset(dataset) # provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True}) provider = doc["provider_name"].lower() elif provider: provider_doc = queries.get_provider(provider, { "slug": True, "name": True, "enable": True }) dataset_doc = queries.col_datasets().find_one( { "provider_name": provider_doc["name"], "enable": True }, {"slug": True}, ) dataset = dataset_doc["slug"] ctx = { "selectedProvider": provider, "selectedDataset": dataset, } return render_template("explorer.html", **ctx)
def dataset_with_slug(slug): dataset = queries.get_dataset(slug, {"_id": False}) is_modal = request.args.get('modal', default=0, type=int) provider = queries.col_providers().find_one({"name": dataset['provider_name']}, {"metadata": False}) #count_series = queries.col_series().count({"provider_name": dataset['provider_name'], # "dataset_code": dataset['dataset_code']}) url_provider = url_for('.explorer_p', provider=provider["slug"]) url_dataset = url_for('.explorer_d', dataset=dataset["slug"]) url_dataset_direct = url_for('.dataset-by-slug', slug=dataset["slug"], _external=True) return render_template("dataset-unit-modal.html", is_modal=is_modal, url_provider=url_provider, url_dataset=url_dataset, url_dataset_direct=url_dataset_direct, dataset=dataset, provider=provider, #count=count_series )
def ajax_explorer_datas(): limit = request.args.get('limit', default=100, type=int) if limit > 1000: limit = 1000 #7 669 115 octets pour 1000 series projection = { "_id": False, "dimensions": False, "attributes": False, "values.revisions": False, "notes": False, "tags": False } query = OrderedDict() provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') #series_slug = request.args.get('series') search = request.args.get('search') is_eurostat = False if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] #query["dataset_code"] = dataset["dataset_code"] if dataset["provider_name"] == "EUROSTAT": is_eurostat = True elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] if provider["name"] == "EUROSTAT": is_eurostat = True if search: query["$text"] = {"$search": search.strip()} projection['score'] = {'$meta': 'textScore'} disabled_datasets = [] if dataset_slug: query["dataset_code"] = dataset["dataset_code"] else: ds_enabled_query = {"enable": False} if "provider_name" in query: ds_enabled_query["provider_name"] = query["provider_name"] disabled_datasets = [ doc["dataset_code"] for doc in queries.col_datasets().find( ds_enabled_query, {"dataset_code": True}) ] query = complex_queries_series(query) cursor = queries.col_series().find(dict(query), projection) if search: cursor = cursor.sort([('score', {'$meta': 'textScore'})]) #else: # query = {"slug": series_slug} # cursor = queries.col_series().find(dict(query), projection) if limit: cursor = cursor.limit(limit) if is_eurostat: count = 0 else: count = cursor.count() series_list = [doc for doc in cursor] rows = [] for s in series_list: if disabled_datasets and s["dataset_code"] in disabled_datasets: continue if not "version" in s: s["version"] = 0 s['start_date'] = s["values"][0]["period"] s['end_date'] = s["values"][-1]["period"] values = [{ "period": v["period"], "value": v["value"] } for v in s['values']] del s["values"] s["values"] = values s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) #s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug']) #TODO: s['url_dataset'] = url_for('.dataset', id=s['_id']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] rows.append(s) return json_tools.json_response(rows, {"total": count})
def ajax_explorer_datas(): limit = request.args.get('limit', default=100, type=int) if limit > 1000: limit = 1000 #7 669 115 octets pour 1000 series projection = { "_id": False, "dimensions": False, "attributes": False, "values.revisions": False, "notes": False, "tags": False } query = OrderedDict() provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') #series_slug = request.args.get('series') search = request.args.get('search') is_eurostat = False if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] #query["dataset_code"] = dataset["dataset_code"] if dataset["provider_name"] == "EUROSTAT": is_eurostat = True elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] if provider["name"] == "EUROSTAT": is_eurostat = True if search: query["$text"] = {"$search": search.strip()} projection['score'] = {'$meta': 'textScore'} disabled_datasets = [] if dataset_slug: query["dataset_code"] = dataset["dataset_code"] else: ds_enabled_query = {"enable": False} if "provider_name" in query: ds_enabled_query["provider_name"] = query["provider_name"] disabled_datasets = [doc["dataset_code"] for doc in queries.col_datasets().find(ds_enabled_query, {"dataset_code": True})] query = complex_queries_series(query) cursor = queries.col_series().find(dict(query), projection) if search: cursor = cursor.sort([('score', {'$meta': 'textScore'})]) #else: # query = {"slug": series_slug} # cursor = queries.col_series().find(dict(query), projection) if limit: cursor = cursor.limit(limit) if is_eurostat: count = 0 else: count = cursor.count() series_list = [doc for doc in cursor] rows = [] for s in series_list: if disabled_datasets and s["dataset_code"] in disabled_datasets: continue if not "version" in s: s["version"] = 0 s['start_date'] = s["values"][0]["period"] s['end_date'] = s["values"][-1]["period"] values = [{"period": v["period"], "value": v["value"]} for v in s['values']] del s["values"] s["values"] = values s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) #s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug']) #TODO: s['url_dataset'] = url_for('.dataset', id=s['_id']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] rows.append(s) return json_tools.json_response(rows, {"total": count})