def view_logs(): """ { "_id" : ObjectId("5665a7182d4b25012092ac71"), "message" : "change count for BEA.datasets. old[0] - new[0]", "level" : "INFO", "timestamp" : Timestamp(1449502488, 860), "loggerName" : "widukind_web", "thread" : 74212736, "threadName" : "DummyThread-1", "method" : "upsert", "lineNumber" : 271, "module" : "wsgi", "fileName" : "V:\\git\\cepremap\\src\\widukind-web\\widukind_web\\wsgi.py" } """ is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("admin/logs.html") col = current_app.widukind_db[constants.COL_LOGS] object_list = col.find({}) return current_app.jsonify(list(object_list))
def queries_view(): is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("admin/queries.html") col = current_app.widukind_db[constants.COL_QUERIES] tags = request.args.get('tags') q = {} if tags: tags = tags.split(",") q['tags'] = {"$in": tags} object_list = col.find(q).sort("created", DESCENDING) result = [] for obj in object_list: obj["view"] = url_for(".doc", col=constants.COL_QUERIES, objectid=obj["_id"]) result.append(obj) return current_app.jsonify(result)
def search_in_series(): is_ajax = request.args.get('json') or request.is_xhr form = get_search_form(forms.SearchFormSeries) if form.validate_on_submit(): projection = { "dimensions": False, "attributes": False, "releaseDates": False, "revisions": False, "values": False } kwargs = get_search_datas(form, search_type="series") object_list = search_series_tags(current_app.widukind_db, projection=projection, **kwargs) object_list = convert_series_period(object_list) record_query(query=kwargs, result_count=len(object_list), form=form, tags=["search", "series"]) for s in object_list: s['view'] = url_for('.serie', id=s['_id']) return current_app.jsonify(dict(count=len(object_list), object_list=object_list, query=kwargs)) return render_template('search-series.html', form=form, search_type="series")
def ajax_view_cart(): datas = {"rows": [], "total": 0} projection = { "dimensions": False, "attributes": False, "releaseDates": False, "revisions": False, "values": False, } cart = session.get("cart", None) if cart: series_ids = [ObjectId(c) for c in cart] series = current_app.widukind_db[constants.COL_SERIES].find({"_id": {"$in": series_ids}}, projection=projection) series = convert_series_period(series) for s in series: s['view'] = url_for('.serie', id=s['_id']) datas["rows"].append(s) #datas["total"] = len(datas["rows"]) #pprint(datas) return current_app.jsonify(datas["rows"])
def ajax_tag_prefetch_series(): provider = request.args.get('provider') if not provider: abort(400, "provider is required") dataset = request.args.get('dataset') limit = request.args.get('limit', default=200, type=int) count = request.args.get('count', default=0, type=int) col = current_app.widukind_db[constants.COL_TAGS] #TODO: renvoyer aussi count pour tag(count) query = OrderedDict() query["provider_name"] = {"$in": [provider.upper()]} if dataset: query["datasets"] = {"$in": [dataset]} if count: query["count"] = {"$gte": count} projection = { "_id": False, "name": True, "count": True, "count_datasets": True, "count_series": True } docs = col.find(query, projection=projection).sort("count", DESCENDING).limit(limit) return current_app.jsonify(list(docs))
def ajax_add_cart(): _id = request.args.get('id') cart = session.get("cart", []) if not _id in cart: cart.append(_id) flash("Series add to cart.", "success") else: flash("Series is already in the cart.", "warning") session["cart"] = cart return current_app.jsonify(dict(count=len(session["cart"])))
def ajax_cart_add(): slug = request.args.get('slug') cart = session.get("cart", []) if not slug in cart: cart.append(slug) msg = {"msg": "Series add to cart.", "category": "success"} else: msg = {"msg": "Series is already in the cart.", "category": "warning"} session["cart"] = cart return current_app.jsonify(dict(notify=msg, count=len(session["cart"])))
def tag_prefetch_series(): """ TODO: notion de selection provider ? >>> len(db.tags.series.distinct("name", {"count": {"$gte": 20}})) 576 >>> len(db.tags.series.distinct("name", {"count": {"$gte": 10}})) 615 >>> len(db.tags.series.distinct("name", {"count": {"$gte": 50}})) 460 >>> len(db.tags.series.distinct("name", {"count": {"$gte": 5}})) 666 >>> len(db.tags.series.distinct("name", {"count": {"$gte": 4}})) 676 >>> len(db.tags.series.distinct("name", {"count": {"$gte": 3}})) 699 >>> len(db.tags.series.distinct("name", {"count": {"$gte": 2}})) 712 """ provider = request.args.get('provider') limit = request.args.get('limit', default=200, type=int) col = current_app.widukind_db[constants.COL_TAGS_SERIES] query = {"count": {"$gte": 5}} if provider: query["providers.name"] = {"$in": [provider]} tags = col.distinct("name", query) return current_app.jsonify(tags) projection = {"_id": False, "providers": False} query = {} docs = col.find(query, projection=projection).sort("count", DESCENDING).limit(limit) tags = [doc['name'] for doc in docs] #print(tags) return current_app.jsonify(tags)
def ajax_cart_view(): is_ajax = request.args.get('json') #or request.is_xhr if not is_ajax: return render_template("series-cart.html") datas = {"rows": None, "total": 0} projection = { "dimensions": False, "attributes": False, "release_dates": False, "revisions": False, "values": False, } cart = session.get("cart", None) if cart: series_slug = [c for c in cart] series = queries.col_series().find({"slug": { "$in": series_slug }}, projection=projection) docs = list(series) for s in docs: if not "version" in s: s["version"] = 0 s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_remove'] = url_for('.ajax-cart-remove', slug=s['slug']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] datas["rows"] = docs return current_app.jsonify(datas["rows"])
def ajax_cart_remove(): slug = request.args.get('slug') is_all = slug == "all" cart = session.get("cart", []) if is_all: cart = [] msg = {"msg": "All series deleted in cart.", "category": "success"} else: if slug in cart: cart.remove(slug) msg = {"msg": "Series remove from cart.", "category": "success"} else: msg = {"msg": "Series not in the cart.", "category": "warning"} session["cart"] = cart return current_app.jsonify(dict(notify=msg, count=len(session["cart"])))
def contacts_view(): is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("admin/contacts.html") col = queries.col_contact() object_list = col.find({}).sort("created", DESCENDING) result = [] for obj in object_list: obj["view"] = url_for(".doc", col=constants.COL_CONTACT, objectid=obj["_id"]) result.append(obj) return current_app.jsonify(result)
def category_tree_view(provider): is_ajax = request.args.get('json') or request.is_xhr tree = _category_tree(provider) #dataset_codes = current_app.widukind_db[constants.COL_DATASETS].distinct("datasetCode", {"provider": provider}) dataset_projection = {"_id": True, "datasetCode": True} dataset_codes = {} for doc in current_app.widukind_db[constants.COL_DATASETS].find({"provider": provider}, dataset_projection): dataset_codes[doc['datasetCode']] = doc['_id'] if is_ajax: return current_app.jsonify(tree) return render_template('categories.html', tree=tree, dataset_codes=dataset_codes)
def ajax_cart_view(): is_ajax = request.args.get('json') #or request.is_xhr if not is_ajax: return render_template("series-cart.html") datas = {"rows": None, "total": 0} projection = { "dimensions": False, "attributes": False, "release_dates": False, "revisions": False, "values": False, } cart = session.get("cart", None) if cart: series_slug = [c for c in cart] series = queries.col_series().find({"slug": {"$in": series_slug}}, projection=projection) docs = list(series) for s in docs: if not "version" in s: s["version"] = 0 s['view'] = url_for('.series-by-slug', slug=s['slug'], modal=1) dataset_slug = slugify("%s-%s" % (s["provider_name"], s["dataset_code"]), word_boundary=False, save_order=True) s['view_dataset'] = url_for('.dataset-by-slug', slug=dataset_slug, modal=1) s['dataset_slug'] = dataset_slug s['export_csv'] = url_for('.export-series-csv', slug=s['slug']) s['url_series_plot'] = url_for('.ajax_series_plot', slug=s['slug']) s['url_cart_remove'] = url_for('.ajax-cart-remove', slug=s['slug']) s['frequency_txt'] = s['frequency'] if s['frequency'] in constants.FREQUENCIES_DICT: s['frequency_txt'] = constants.FREQUENCIES_DICT[s['frequency']] datas["rows"] = docs return current_app.jsonify(datas["rows"])
def html_datasets(provider=None): is_ajax = request.args.get('json') or request.is_xhr projection = { "dimensionList": False, "attributeList": False, } provider_doc = None if provider: provider_doc = current_app.widukind_db[constants.COL_PROVIDERS].find_one({"name": provider}) if not is_ajax: return render_template("datasets.html", provider=provider_doc) query = {} if provider: query['provider'] = provider datasets = current_app.widukind_db[constants.COL_DATASETS].find(query, projection=projection, sort=[('lastUpdate', DESCENDING)]) count, objects = filter_query(datasets) if is_ajax: datas = { "total": count, "rows": [] } for o in objects: o['view'] = url_for('.dataset', id=o['_id']) o['series'] = url_for('.series_with_datasetCode', provider=o['provider'], datasetCode=o['datasetCode']) doc_href = o.get('docHref', None) if doc_href and not doc_href.lower().startswith('http'): o['docHref'] = None datas["rows"].append(o) return current_app.jsonify(datas)
def queries(): is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("admin/queries.html") col = current_app.widukind_db[constants.COL_QUERIES] tags = request.args.get('tags') q = {} if tags: tags = tags.split(",") q['tags'] = {"$in": tags} object_list = col.find(q).sort("created", DESCENDING) return current_app.jsonify(list(object_list))
def last_series(): """ > projection sur 1 élément du champs array > sort sur cet élément d'un champs array pprint(list(db.series.find({}, projection={"key": True, "_id": False, "releaseDates": {"$slice": 1}}).limit(20).sort([("releaseDates.0", -1)]))) """ LIMIT = 20 is_ajax = request.args.get('json') or request.is_xhr query = {} projection = { "dimensions": False, "attributes": False, "releaseDates": False, "revisions": False, } if not is_ajax: return render_template("last_series.html") _series = current_app.widukind_db[constants.COL_SERIES].find(query, projection=projection) count, series = filter_query(_series, limit=LIMIT) if is_ajax: datas = { "total": count, "rows": [] } series = convert_series_period(series) for s in series: s['view'] = url_for('.serie', id=s['_id']) s['export_csv'] = url_for('download.series_csv', provider=s['provider'], datasetCode=s['datasetCode'], key=s['key']) s['view_graphic'] = url_for('.series_plot', id=s['_id']) datas["rows"].append(s) return current_app.jsonify(datas)
def ajax_tag_prefetch_series(): provider = request.args.get('provider') if not provider: abort(400, "provider is required") dataset = request.args.get('dataset') limit = request.args.get('limit', default=200, type=int) count = request.args.get('count', default=0, type=int) col = current_app.widukind_db[constants.COL_TAGS] #TODO: renvoyer aussi count pour tag(count) query = OrderedDict() query["provider_name"] = {"$in": [provider.upper()]} if dataset: query["datasets"] = {"$in": [dataset]} if count: query["count"] = {"$gte": count} projection = {"_id": False, "name": True, "count": True, "count_datasets": True, "count_series": True} docs = col.find(query, projection=projection).sort("count", DESCENDING).limit(limit) return current_app.jsonify(list(docs))
def last_datasets(): LIMIT = 20 is_ajax = request.args.get('json') or request.is_xhr query = {} projection = { "dimensionList": False, "attributeList": False, } if not is_ajax: return render_template("last_datasets.html") _object_list = current_app.widukind_db[constants.COL_DATASETS].find(query, projection=projection) count, object_list = filter_query(_object_list, limit=LIMIT, execute=True) import time time.sleep(2) if is_ajax: datas = { "total": count, "rows": [] } for s in object_list: s['view'] = url_for('.dataset', id=s['_id']) doc_href = s.get('docHref', None) if doc_href and not doc_href.lower().startswith('http'): s['docHref'] = None datas["rows"].append(s) # pagination client - return rows only return current_app.jsonify(datas["rows"])
def series_with_slug(slug, version): """ Dans tous les cas: - charger la version latest - charger toutes les révisions antérieurs à la version latest """ is_modal = request.args.get('modal', default=0, type=int) is_debug = request.args.get('debug') #_version = request.args.get('version', default="latest") is_latest = True query = {"slug": slug} '''Load always latest series from col series''' series_latest = queries.col_series().find_one(query) if not series_latest: abort(404) if version >= 0 and version != series_latest['version']: query['version'] = version store = queries.col_series_archives().find_one(query) if not store: abort(404) series = series_archives_load(store) is_latest = False else: series = series_latest provider = queries.col_providers().find_one( {"name": series_latest['provider_name']}, {"metadata": False}) if not provider: abort(404) if provider["enable"] is False: abort(307) dataset = queries.col_datasets().find_one( { 'provider_name': series_latest['provider_name'], "dataset_code": series_latest['dataset_code'] }, {"metadata": False}) if not dataset: abort(404) if dataset["enable"] is False: abort(307) if is_debug: '''debug mode''' result_provider = render_template_string("{{ provider|pprint|safe }}", provider=provider) result_dataset = render_template_string("{{ dataset|pprint|safe }}", dataset=dataset) result_series = render_template_string("{{ series|pprint|safe }}", series=series) return current_app.jsonify( dict(provider=result_provider, dataset=result_dataset, series=result_series)) '''Load revisions < current version''' revisions = [] if "version" in series: query_revisions = {"slug": slug, "version": {"$lt": series["version"]}} count_values = len(series['values']) for store in queries.col_series_archives().find(query_revisions).sort( 'version', DESCENDING): series_rev = series_archives_load(store) values = series_rev['values'] empty_element = count_values - len(values) values.reverse() for i in range(empty_element): values.insert(0, None) revisions.append({ "last_update_ds": series_rev['last_update_ds'], "version": series_rev['version'], "values": values, "name": series_rev["name"], "url": url_for('.series-by-slug-version', slug=slug, version=series_rev['version']) }) else: series["version"] = 0 if not "last_update_ds" in series: series["last_update_ds"] = dataset["last_update"] series["last_update_widu"] = dataset["last_update"] #view_explorer = url_for('.explorer_s', series=slug, _external=True) url_provider = url_for('.explorer_p', provider=provider["slug"]) url_dataset = url_for('.explorer_d', dataset=dataset["slug"]) url_dataset_direct = url_for('.dataset-by-slug', slug=dataset["slug"], _external=True) url_series = url_for('.series-by-slug-version', slug=slug, version=series["version"], _external=True) url_series_latest = url_for('.series-by-slug-version', slug=slug, version=series_latest["version"]) url_series_plot = url_for('.ajax_series_plot', slug=slug) url_export_csv = url_for('.export-series-csv', slug=slug) dimension_filter = ".".join( [series["dimensions"][key] for key in dataset["dimension_keys"]]) result = render_template( "series-unit-modal.html", url_provider=url_provider, url_dataset=url_dataset, url_dataset_direct=url_dataset_direct, url_series=url_series, url_series_latest=url_series_latest, url_series_plot=url_series_plot, url_export_csv=url_export_csv, series=series, is_modal=is_modal, provider=provider, dataset=dataset, is_latest=is_latest, revisions=revisions, #max_version=max_version, #view_explorer=view_explorer, dimension_filter=dimension_filter.upper(), #is_reverse=is_reverse, #obs_attributes_keys=list(set(obs_attributes_keys)), #obs_attributes_values=list(set(obs_attributes_values)), #revision_dates=list(set(revision_dates)), #max_revisions=max_revisions ) return result
def series_with_slug(slug, version): """ Dans tous les cas: - charger la version latest - charger toutes les révisions antérieurs à la version latest """ is_modal = request.args.get('modal', default=0, type=int) is_debug = request.args.get('debug') #_version = request.args.get('version', default="latest") is_latest = True query = {"slug": slug} '''Load always latest series from col series''' series_latest = queries.col_series().find_one(query) if not series_latest: abort(404) if version >= 0 and version != series_latest['version']: query['version'] = version store = queries.col_series_archives().find_one(query) if not store: abort(404) series = series_archives_load(store) is_latest = False else: series = series_latest provider = queries.col_providers().find_one({"name": series_latest['provider_name']}, {"metadata": False}) if not provider: abort(404) if provider["enable"] is False: abort(307) dataset = queries.col_datasets().find_one({'provider_name': series_latest['provider_name'], "dataset_code": series_latest['dataset_code']}, {"metadata": False}) if not dataset: abort(404) if dataset["enable"] is False: abort(307) if is_debug: '''debug mode''' result_provider = render_template_string("{{ provider|pprint|safe }}", provider=provider) result_dataset = render_template_string("{{ dataset|pprint|safe }}", dataset=dataset) result_series = render_template_string("{{ series|pprint|safe }}", series=series) return current_app.jsonify(dict(provider=result_provider, dataset=result_dataset, series=result_series)) '''Load revisions < current version''' revisions = [] if "version" in series: query_revisions = {"slug": slug, "version": {"$lt": series["version"]}} count_values = len(series['values']) for store in queries.col_series_archives().find(query_revisions).sort('version', DESCENDING): series_rev = series_archives_load(store) values = series_rev['values'] empty_element = count_values - len(values) values.reverse() for i in range(empty_element): values.insert(0, None) revisions.append({ "last_update_ds": series_rev['last_update_ds'], "version": series_rev['version'], "values": values, "name": series_rev["name"], "url": url_for('.series-by-slug-version', slug=slug, version=series_rev['version'])}) else: series["version"] = 0 if not "last_update_ds" in series: series["last_update_ds"] = dataset["last_update"] series["last_update_widu"] = dataset["last_update"] #view_explorer = url_for('.explorer_s', series=slug, _external=True) url_provider = url_for('.explorer_p', provider=provider["slug"]) url_dataset = url_for('.explorer_d', dataset=dataset["slug"]) url_dataset_direct = url_for('.dataset-by-slug', slug=dataset["slug"], _external=True) url_series = url_for('.series-by-slug-version', slug=slug, version=series["version"], _external=True) url_series_latest = url_for('.series-by-slug-version', slug=slug, version=series_latest["version"]) url_series_plot = url_for('.ajax_series_plot', slug=slug) url_export_csv = url_for('.export-series-csv', slug=slug) dimension_filter = ".".join([series["dimensions"][key] for key in dataset["dimension_keys"]]) result = render_template( "series-unit-modal.html", url_provider=url_provider, url_dataset=url_dataset, url_dataset_direct=url_dataset_direct, url_series=url_series, url_series_latest=url_series_latest, url_series_plot=url_series_plot, url_export_csv=url_export_csv, series=series, is_modal=is_modal, provider=provider, dataset=dataset, is_latest=is_latest, revisions=revisions, #max_version=max_version, #view_explorer=view_explorer, dimension_filter=dimension_filter.upper(), #is_reverse=is_reverse, #obs_attributes_keys=list(set(obs_attributes_keys)), #obs_attributes_values=list(set(obs_attributes_values)), #revision_dates=list(set(revision_dates)), #max_revisions=max_revisions ) return result
def html_series(provider=None, datasetCode=None): is_ajax = request.args.get('json') or request.is_xhr query = {} if provider: query['provider'] = provider if datasetCode: query['datasetCode'] = datasetCode search_filter = None if request.args.get('filter'): search_filter = json.loads(request.args.get('filter')) if 'startDate' in search_filter: """ > Manque la fréquence pd.Period("1995", freq="A").ordinal """ #print("search_filter : ", search_filter, type(search_filter)) search_tags = request.args.get('tags') if search_tags: tags = [t.strip().lower() for t in search_tags.split()] query["tags"] = {"$all": tags} projection = { "dimensions": False, "attributes": False, "releaseDates": False, "revisions": False, "values": False, "dimensions": False, "notes": False } if search_filter: filter.update(search_filter) #print("query : ", query) if not is_ajax: dataset = None if datasetCode: dataset = current_app.widukind_db[constants.COL_DATASETS].find_one(query) provider_doc = None if provider: provider_doc = current_app.widukind_db[constants.COL_PROVIDERS].find_one({"name": provider}) return render_template("series.html", provider=provider_doc, dataset=dataset) series = current_app.widukind_db[constants.COL_SERIES].find(query, projection=projection) count, series = filter_query(series) if is_ajax: datas = { "total": count, "rows": [] } series = convert_series_period(series) for s in series: s['view'] = url_for('.serie', id=s['_id']) s['export_csv'] = url_for('download.series_csv', provider=s['provider'], datasetCode=s['datasetCode'], key=s['key']) #s['export_csv'] = url_for('download.series_csv', provider=s['provider'], datasetCode=s['datasetCode'], key=s['key']) s['view_graphic'] = url_for('.series_plot', id=s['_id']) #TODO: s['url_dataset'] = url_for('.dataset', id=s['_id']) datas["rows"].append(s) return current_app.jsonify(datas)