def datasets_last_update(): is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("datasets-last-update.html") query = { "$or": [{"count_inserts": {"$gt": 0}}, {"count_updates": {"$gt": 0}}] } startDate = arrow.utcnow().replace(days=-1).floor("second") query["created"] = {"$gte": startDate.datetime} limit = request.args.get('limit', default=0, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] for row in rows: slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]), word_boundary=False, save_order=True) row["view"] = url_for(".explorer_d", dataset=slug) return json_tools.json_response(rows, {"total": count})
def stats_run_json(): query = {} provider_slug = request.args.get('provider') dataset_slug = request.args.get('dataset') if dataset_slug: dataset = queries.get_dataset(dataset_slug) query["provider_name"] = dataset["provider_name"] query["dataset_code"] = dataset["dataset_code"] elif provider_slug: provider = queries.get_provider(provider_slug) query["provider_name"] = provider["name"] startDate = arrow.get(request.args.get('startDate')).floor('day').datetime endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime query["created"] = {"$gte": startDate, "$lte": endDate} limit = request.args.get('limit', default=100, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] """ for row in rows: row["view"] = url_for("views.dataset-by-code", provider_name=row["provider_name"], dataset_code=row["dataset_code"]) """ return json_tools.json_response(rows, {"total": count})
def atom_feed(): from werkzeug.contrib.atom import AtomFeed now = arrow.utcnow().datetime feed = AtomFeed("Widukind", feed_url=request.url, #url=request.host_url, #updated=now, subtitle="Updated datasets - Last 24 hours" ) query = { "$or": [{"count_inserts": {"$gt": 0}}, {"count_updates": {"$gt": 0}}] } startDate = arrow.utcnow().replace(days=-1).floor("second") query["created"] = {"$gte": startDate.datetime} limit = request.args.get('limit', default=0, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] slugs = [] for row in rows: slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]), word_boundary=False, save_order=True) slugs.append((row, slug)) query_dataset = {"slug": {"$in": [s[1] for s in slugs]}} projection = {"metadata": False, "concepts": False, "codelists": False} datasets = {doc["slug"]: doc for doc in queries.col_datasets().find(query_dataset, projection)} for row, slug in slugs: dataset = datasets.get(slug) if not dataset["enable"]: continue url = url_for("views.explorer_d", dataset=slug, _external=True) #content = """ #<p>Updated date from provider : %(last_update)s</p> #""" feed.add(title="%s - %s" % (row["provider_name"], row["dataset_code"]), summary=dataset["name"], #content=content % dataset, #content_type="html", url=url, id=slug, updated=row["created"], #dataset["last_update"], published=dataset["download_last"] ) return feed.get_response()
def datasets_last_update(): is_ajax = request.args.get('json') or request.is_xhr if not is_ajax: return render_template("datasets-last-update.html") query = { "$or": [{ "count_inserts": { "$gt": 0 } }, { "count_updates": { "$gt": 0 } }] } startDate = arrow.utcnow().replace(days=-1).floor("second") query["created"] = {"$gte": startDate.datetime} limit = request.args.get('limit', default=0, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) count = cursor.count() cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] for row in rows: slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]), word_boundary=False, save_order=True) row["view"] = url_for(".explorer_d", dataset=slug) return json_tools.json_response(rows, {"total": count})
def atom_feed(): from werkzeug.contrib.atom import AtomFeed now = arrow.utcnow().datetime feed = AtomFeed( "DB.nomics", feed_url=request.url, #url=request.host_url, #updated=now, subtitle="Updated datasets - Last 24 hours") query = { "$or": [{ "count_inserts": { "$gt": 0 } }, { "count_updates": { "$gt": 0 } }] } startDate = arrow.utcnow().replace(days=-1).floor("second") query["created"] = {"$gte": startDate.datetime} limit = request.args.get('limit', default=0, type=int) cursor = queries.col_stats_run().find(query) if limit: cursor = cursor.limit(limit) cursor = cursor.sort("created", -1) rows = [doc for doc in cursor] slugs = [] for row in rows: slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]), word_boundary=False, save_order=True) slugs.append((row, slug)) query_dataset = {"slug": {"$in": [s[1] for s in slugs]}} projection = {"metadata": False, "concepts": False, "codelists": False} datasets = { doc["slug"]: doc for doc in queries.col_datasets().find(query_dataset, projection) } for row, slug in slugs: dataset = datasets.get(slug) if not dataset["enable"]: continue url = url_for("views.explorer_d", dataset=slug, _external=True) #content = """ #<p>Updated date from provider : %(last_update)s</p> #""" feed.add( title="%s - %s" % (row["provider_name"], row["dataset_code"]), summary=dataset["name"], #content=content % dataset, #content_type="html", url=url, id=slug, updated=row["created"], #dataset["last_update"], published=dataset["download_last"]) return feed.get_response()