def explorer_view(provider=None, dataset=None, series=None): """ http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer/insee http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer http://127.0.0.1:8081/views """ if not dataset and not provider: provider = current_app.config.get('DEFAULT_PROVIDER', None) dataset = current_app.config.get('DEFAULT_DATASET', None) if dataset: doc = queries.get_dataset(dataset) #provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True}) provider = doc["provider_name"].lower() elif provider: provider_doc = queries.get_provider(provider, {"slug": True, "name": True, "enable": True}) dataset_doc = queries.col_datasets().find_one({"provider_name": provider_doc["name"], "enable": True}, {"slug": True}) dataset = dataset_doc["slug"] ctx = { "selectedProvider": provider, "selectedDataset": dataset, } return render_template("explorer.html", **ctx)
def ajax_datasets_list(provider): """ TODO: covered query provider_name + enable + dataset_code + name + slug ??? partialfilter sur enable = True """ provider_doc = queries.get_provider(provider) query = {'provider_name': provider_doc["name"]} projection = { "_id": False, "enable": True, "dataset_code": True, "name": True, "slug": True } is_meta = request.args.get('is_meta', type=int, default=0) == 1 if is_meta: projection["metadata"] = True docs = [ doc for doc in queries.col_datasets().find(query, projection) if doc["enable"] is True ] return json_tools.json_response(docs)
def stats_run_json(): query = {} provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] query["dataset_code"] = dataset["dataset_code"] elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] startDate = arrow.get(request.args.get('startDate')).floor('day').datetime endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime query["created"] = {"$gte": startDate, "$lte": endDate} limit = request.args.get('limit', default=100, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] """ for row in rows: row["view"] = url_for("views.dataset-by-code", provider_name=row["provider_name"], dataset_code=row["dataset_code"]) """ return json_tools.json_response(rows, {"total": count})
def ajax_tree_view(provider=None): provider = provider or request.args.get('provider') if not provider: abort(404, "provider is required") _provider = queries.get_provider(provider) provider_name = _provider["name"] query = {"provider_name": provider_name, "enable": True} cursor = queries.col_categories().find(query, {"_id": False}) cursor = cursor.sort([("position", 1), ("category_code", 1)]) categories = OrderedDict([(doc["category_code"], doc) for doc in cursor]) ds_codes = [] for_remove = [] for cat in categories.values(): if cat.get("parent"): parent = categories[cat.get("parent")] if not "children" in parent: parent["children"] = [] parent["children"].append(cat) for_remove.append(cat["category_code"]) if cat.get("datasets"): for ds in cat.get("datasets"): ds_codes.append(ds["dataset_code"]) for r in for_remove: categories.pop(r) ds_query = { 'provider_name': provider_name, "enable": True, "dataset_code": { "$in": list(set(ds_codes)) } } ds_projection = {"_id": True, "dataset_code": True, "slug": True} cursor = queries.col_datasets().find(ds_query, ds_projection) dataset_codes = {} for doc in cursor: dataset_codes[doc['dataset_code']] = { "slug": doc['slug'], "url": url_for('.dataset-by-slug', slug=doc["slug"]) } context = dict(provider=_provider, categories=categories, dataset_codes=dataset_codes) data = render_template("datatree_ajax.html", **context) response = current_app.response_class(data, mimetype='application/javascript') response.status_code = 200 response.make_conditional(request) return response
def ajax_tree_view(provider=None): provider = provider or request.args.get('provider') if not provider: abort(404, "provider is required") _provider = queries.get_provider(provider) provider_name = _provider["name"] query = {"provider_name": provider_name, "enable": True} cursor = queries.col_categories().find(query, {"_id": False}) cursor = cursor.sort([("position", 1), ("category_code", 1)]) categories = OrderedDict([(doc["category_code"], doc ) for doc in cursor]) ds_codes = [] for_remove = [] for cat in categories.values(): if cat.get("parent"): parent = categories[cat.get("parent")] if not "children" in parent: parent["children"] = [] parent["children"].append(cat) for_remove.append(cat["category_code"]) if cat.get("datasets"): for ds in cat.get("datasets"): ds_codes.append(ds["dataset_code"]) for r in for_remove: categories.pop(r) ds_query = {'provider_name': provider_name, "enable": True, "dataset_code": {"$in": list(set(ds_codes))}} ds_projection = {"_id": True, "dataset_code": True, "slug": True} cursor = queries.col_datasets().find(ds_query, ds_projection) dataset_codes = {} for doc in cursor: dataset_codes[doc['dataset_code']] = { "slug": doc['slug'], "url": url_for('.dataset-by-slug', slug=doc["slug"]) } context = dict( provider=_provider, categories=categories, dataset_codes=dataset_codes ) data = render_template("datatree_ajax.html", **context) response = current_app.response_class(data, mimetype='application/javascript') response.status_code = 200 response.make_conditional(request) return response
def ajax_datasets_list(provider): """ TODO: covered query provider_name + enable + dataset_code + name + slug ??? partialfilter sur enable = True """ provider_doc = queries.get_provider(provider) query = {'provider_name': provider_doc["name"]} projection = {"_id": False, "enable": True, "dataset_code": True, "name": True, "slug": True} is_meta = request.args.get('is_meta', type=int, default=0) == 1 if is_meta: projection["metadata"] = True docs = [doc for doc in queries.col_datasets().find(query, projection) if doc["enable"] is True] return json_tools.json_response(docs)
def explorer_view(provider=None, dataset=None, series=None): """ http://127.0.0.1:8081/views/explorer/dataset/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer/insee http://127.0.0.1:8081/views/explorer/insee/insee-cna-2005-ere-a88 http://127.0.0.1:8081/views/explorer http://127.0.0.1:8081/views """ if not dataset and not provider: provider = current_app.config.get('DEFAULT_PROVIDER', None) dataset = current_app.config.get('DEFAULT_DATASET', None) if dataset: doc = queries.get_dataset(dataset) # provider_doc = queries.col_providers().find_one({"name": doc["provider_name"]},{"slug": True}) provider = doc["provider_name"].lower() elif provider: provider_doc = queries.get_provider(provider, { "slug": True, "name": True, "enable": True }) dataset_doc = queries.col_datasets().find_one( { "provider_name": provider_doc["name"], "enable": True }, {"slug": True}, ) dataset = dataset_doc["slug"] ctx = { "selectedProvider": provider, "selectedDataset": dataset, } return render_template("explorer.html", **ctx)
def ajax_explorer_datas(): limit = request.args.get('limit', default=100, type=int) if limit > 1000: limit = 1000 #7 669 115 octets pour 1000 series projection = { "_id": False, "dimensions": False, "attributes": False, "values.revisions": False, "notes": False, "tags": False } query = OrderedDict() provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') #series_slug = request.args.get('series') search = request.args.get('search') is_eurostat = False if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] #query["dataset_code"] = dataset["dataset_code"] if dataset["provider_name"] == "EUROSTAT": is_eurostat = True elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] if provider["name"] == "EUROSTAT": is_eurostat = True if search: query["$text"] = {"$search": search.strip()} projection['score'] = {'$meta': 'textScore'} disabled_datasets = [] if dataset_slug: query["dataset_code"] = dataset["dataset_code"] else: ds_enabled_query = {"enable": False} if "provider_name" in query: ds_enabled_query["provider_name"] = query["provider_name"] disabled_datasets = [ doc["dataset_code"] for doc in queries.col_datasets().find( ds_enabled_query, {"dataset_code": True}) ] query = complex_queries_series(query) cursor = queries.col_series().find(dict(query), projection) if search: cursor = cursor.sort([('score', {'$meta': 'textScore'})]) #else: # query = {"slug": series_slug} # cursor = queries.col_series().find(dict(query), projection) if limit: cursor = cursor.limit(limit) if is_eurostat: count = 0 else: count = cursor.count() series_list = [doc for doc in cursor] rows = [] for s in series_list: if disabled_datasets and s["dataset_code"] in disabled_datasets: continue if not "version" in s: s["version"] = 0 s['start_date'] = s["values"][0]["period"] s['end_date'] = s["values"][-1]["period"] values = [{ "period": v["period"], "value": v["value"] } for v in s['values']] del s["values"] s["values"] = values s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) #s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug']) #TODO: s['url_dataset'] = url_for('.dataset', id=s['_id']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] rows.append(s) return json_tools.json_response(rows, {"total": count})
def ajax_explorer_datas(): limit = request.args.get('limit', default=100, type=int) if limit > 1000: limit = 1000 #7 669 115 octets pour 1000 series projection = { "_id": False, "dimensions": False, "attributes": False, "values.revisions": False, "notes": False, "tags": False } query = OrderedDict() provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') #series_slug = request.args.get('series') search = request.args.get('search') is_eurostat = False if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] #query["dataset_code"] = dataset["dataset_code"] if dataset["provider_name"] == "EUROSTAT": is_eurostat = True elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] if provider["name"] == "EUROSTAT": is_eurostat = True if search: query["$text"] = {"$search": search.strip()} projection['score'] = {'$meta': 'textScore'} disabled_datasets = [] if dataset_slug: query["dataset_code"] = dataset["dataset_code"] else: ds_enabled_query = {"enable": False} if "provider_name" in query: ds_enabled_query["provider_name"] = query["provider_name"] disabled_datasets = [doc["dataset_code"] for doc in queries.col_datasets().find(ds_enabled_query, {"dataset_code": True})] query = complex_queries_series(query) cursor = queries.col_series().find(dict(query), projection) if search: cursor = cursor.sort([('score', {'$meta': 'textScore'})]) #else: # query = {"slug": series_slug} # cursor = queries.col_series().find(dict(query), projection) if limit: cursor = cursor.limit(limit) if is_eurostat: count = 0 else: count = cursor.count() series_list = [doc for doc in cursor] rows = [] for s in series_list: if disabled_datasets and s["dataset_code"] in disabled_datasets: continue if not "version" in s: s["version"] = 0 s['start_date'] = s["values"][0]["period"] s['end_date'] = s["values"][-1]["period"] values = [{"period": v["period"], "value": v["value"]} for v in s['values']] del s["values"] s["values"] = values s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) #s["view_explorer"] = url_for('.explorer_s', series=s['slug'], _external=True) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_add'] = url_for('.ajax-cart-add', slug=s['slug']) #TODO: s['url_dataset'] = url_for('.dataset', id=s['_id']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] rows.append(s) return json_tools.json_response(rows, {"total": count})