def datasets_last_update():
is_ajax = request.args.get('json') or request.is_xhr
if not is_ajax:
return render_template("datasets-last-update.html")
query = {
"$or": [{"count_inserts": {"$gt": 0}}, {"count_updates": {"$gt": 0}}]
}
startDate = arrow.utcnow().replace(days=-1).floor("second")
query["created"] = {"$gte": startDate.datetime}
limit = request.args.get('limit', default=0, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
for row in rows:
slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]),
word_boundary=False, save_order=True)
row["view"] = url_for(".explorer_d", dataset=slug)
return json_tools.json_response(rows, {"total": count})
def stats_run_json():
query = {}
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
query["dataset_code"] = dataset["dataset_code"]
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
startDate = arrow.get(request.args.get('startDate')).floor('day').datetime
endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime
query["created"] = {"$gte": startDate, "$lte": endDate}
limit = request.args.get('limit', default=100, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
"""
for row in rows:
row["view"] = url_for("views.dataset-by-code",
provider_name=row["provider_name"],
dataset_code=row["dataset_code"])
"""
return json_tools.json_response(rows, {"total": count})
def stats_run_json():
query = {}
provider_slug = request.args.get('provider')
dataset_slug = request.args.get('dataset')
if dataset_slug:
dataset = queries.get_dataset(dataset_slug)
query["provider_name"] = dataset["provider_name"]
query["dataset_code"] = dataset["dataset_code"]
elif provider_slug:
provider = queries.get_provider(provider_slug)
query["provider_name"] = provider["name"]
startDate = arrow.get(request.args.get('startDate')).floor('day').datetime
endDate = arrow.get(request.args.get('endDate')).ceil('day').datetime
query["created"] = {"$gte": startDate, "$lte": endDate}
limit = request.args.get('limit', default=100, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
"""
for row in rows:
row["view"] = url_for("views.dataset-by-code",
provider_name=row["provider_name"],
dataset_code=row["dataset_code"])
"""
return json_tools.json_response(rows, {"total": count})
def atom_feed():
from werkzeug.contrib.atom import AtomFeed
now = arrow.utcnow().datetime
feed = AtomFeed("Widukind",
feed_url=request.url,
#url=request.host_url,
#updated=now,
subtitle="Updated datasets - Last 24 hours"
)
query = {
"$or": [{"count_inserts": {"$gt": 0}}, {"count_updates": {"$gt": 0}}]
}
startDate = arrow.utcnow().replace(days=-1).floor("second")
query["created"] = {"$gte": startDate.datetime}
limit = request.args.get('limit', default=0, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
slugs = []
for row in rows:
slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]),
word_boundary=False, save_order=True)
slugs.append((row, slug))
query_dataset = {"slug": {"$in": [s[1] for s in slugs]}}
projection = {"metadata": False, "concepts": False, "codelists": False}
datasets = {doc["slug"]: doc for doc in queries.col_datasets().find(query_dataset, projection)}
for row, slug in slugs:
dataset = datasets.get(slug)
if not dataset["enable"]:
continue
url = url_for("views.explorer_d", dataset=slug, _external=True)
#content = """
#<p>Updated date from provider : %(last_update)s</p>
#"""
feed.add(title="%s - %s" % (row["provider_name"], row["dataset_code"]),
summary=dataset["name"],
#content=content % dataset,
#content_type="html",
url=url,
id=slug,
updated=row["created"], #dataset["last_update"],
published=dataset["download_last"]
)
return feed.get_response()
def datasets_last_update():
is_ajax = request.args.get('json') or request.is_xhr
if not is_ajax:
return render_template("datasets-last-update.html")
query = {
"$or": [{
"count_inserts": {
"$gt": 0
}
}, {
"count_updates": {
"$gt": 0
}
}]
}
startDate = arrow.utcnow().replace(days=-1).floor("second")
query["created"] = {"$gte": startDate.datetime}
limit = request.args.get('limit', default=0, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
count = cursor.count()
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
for row in rows:
slug = slugify("%s-%s" % (row["provider_name"], row["dataset_code"]),
word_boundary=False,
save_order=True)
row["view"] = url_for(".explorer_d", dataset=slug)
return json_tools.json_response(rows, {"total": count})
def atom_feed():
from werkzeug.contrib.atom import AtomFeed
now = arrow.utcnow().datetime
feed = AtomFeed(
"DB.nomics",
feed_url=request.url,
#url=request.host_url,
#updated=now,
subtitle="Updated datasets - Last 24 hours")
query = {
"$or": [{
"count_inserts": {
"$gt": 0
}
}, {
"count_updates": {
"$gt": 0
}
}]
}
startDate = arrow.utcnow().replace(days=-1).floor("second")
query["created"] = {"$gte": startDate.datetime}
limit = request.args.get('limit', default=0, type=int)
cursor = queries.col_stats_run().find(query)
if limit:
cursor = cursor.limit(limit)
cursor = cursor.sort("created", -1)
rows = [doc for doc in cursor]
slugs = []
for row in rows:
slug = slugify("%s-%s" %
(row["provider_name"], row["dataset_code"]),
word_boundary=False,
save_order=True)
slugs.append((row, slug))
query_dataset = {"slug": {"$in": [s[1] for s in slugs]}}
projection = {"metadata": False, "concepts": False, "codelists": False}
datasets = {
doc["slug"]: doc
for doc in queries.col_datasets().find(query_dataset, projection)
}
for row, slug in slugs:
dataset = datasets.get(slug)
if not dataset["enable"]:
continue
url = url_for("views.explorer_d", dataset=slug, _external=True)
#content = """
#<p>Updated date from provider : %(last_update)s</p>
#"""
feed.add(
title="%s - %s" % (row["provider_name"], row["dataset_code"]),
summary=dataset["name"],
#content=content % dataset,
#content_type="html",
url=url,
id=slug,
updated=row["created"], #dataset["last_update"],
published=dataset["download_last"])
return feed.get_response()
