def show_ra(provider):
if not provider in region_map:
return redirect('/')
# get numbers for the RA
service_counts = db.Service.count_types_by_provider()
dataset_counts = db.Dataset.count_types_by_provider()
provider_service_count = service_counts.get(provider, {}).get('_all', 0)
provider_dataset_count = dataset_counts.get(provider, {}).get('_all', 0)
# get tls for this RA
tlds = db.Service.find({
'data_provider': provider,
'active': True
}).distinct('tld')
# get provider extra info from dictionary
pi = provider_info.get(provider, {})
return render_template("show_ra.html",
pi=pi,
provider=provider,
providers=region_map.keys(),
provider_service_count=provider_service_count,
tlds=tlds,
provider_dataset_count=provider_dataset_count)
def view_metadatas(filter_provider):
service_filters = {'active': True}
if filter_provider is not None:
service_filters['data_provider'] = filter_provider
sids = get_service_ids(service_filters)
metadatas, cols, dids = get_metadatas(sids)
# get mappings of services/datasets
services = {
s._id: s
for s in db.Service.find({'_id': {
'$in': list(sids)
}})
}
#datasets = {d._id:d.name for d in db.Dataset.find({'_id':{'$in':list(dids)}})}
return render_template(
"metadatas.html",
metadatas=metadatas,
services=services,
#datasets=datasets,
providers=region_map.keys(),
filters=service_filters,
columns=list(cols))
def view_metadatas(filter_provider):
service_filters = {'active': True}
if filter_provider is not None:
service_filters['data_provider'] = filter_provider
sids = get_service_ids(service_filters)
metadatas, cols, dids = get_metadatas(sids)
# get mappings of services/datasets
services = {
s._id: s
for s in db.Service.find({
'_id': {
'$in': list(sids)
}
})
}
#datasets = {d._id:d.name for d in db.Dataset.find({'_id':{'$in':list(dids)}})}
return render_template(
"metadatas.html",
metadatas=metadatas,
services=services,
#datasets=datasets,
providers=region_map.keys(),
filters=service_filters,
columns=list(cols))
def dashboard():
# provider list
providers = sorted(region_map.keys())
# service counts by provider
counts_by_provider = db.Service.count_types_by_provider_flat()
dataset_counts_by_provider = db.Dataset.count_types_by_provider_flat()
# get list of most recent updates since yesterday
since = datetime.utcnow() - timedelta(hours=24)
stats = list(db.PingLatest.find({'updated':{'$gte':since}}).sort([('updated',-1)]))
upd_datasets = list(db.Dataset.find({'updated':{'$gte':since}}).sort([('updated', -1)]))
services = db.Service.find({'_id':{'$in':[p.service_id for p in stats]}})
services = {s._id:s for s in services}
updates = []
for s in stats:
try:
updates.append({'data_provider':services[s._id].data_provider,
'name': services[s._id].name,
'service_type': services[s._id].service_type,
'update_type': 'ping',
'updated': int(s.updated.strftime('%s')),
'updated_display': prettydate(s.updated),
'data': {'code':s.last_response_code,
'time':s.last_response_time},
'_id': str(s._id),
'url': url_for('show_service', service_id=s._id)})
except KeyError:
app.logger.exception('KeyError updating stats')
for d in upd_datasets:
for s in d.services:
# due to the grouping, we may have older ones in this dataset
if s['updated'] != d.updated:
continue
updates.append({'data_provider':s['data_provider'],
'name':s['name'],
'service_type': s['service_type'],
'update_type':'harvest',
'updated': int(d.updated.strftime('%s')),
'updated_display': prettydate(d.updated),
'data':{},
'id':str(d._id),
'url':url_for('show_dataset', dataset_id=d._id)})
return render_template('index.html',
counts_by_provider=counts_by_provider,
dataset_counts_by_provider=dataset_counts_by_provider,
updates=updates,
providers=providers)
def show_ra(provider):
if not provider in region_map:
return redirect('/')
# get numbers for the RA
service_counts = db.Service.count_types_by_provider()
dataset_counts = db.Dataset.count_types_by_provider()
provider_service_count = service_counts.get(provider, {}).get('_all', 0)
provider_dataset_count = dataset_counts.get(provider, {}).get('_all', 0)
# get tls for this RA
tlds = db.Service.find({'data_provider':provider, 'active':True}).distinct('tld')
# get provider extra info from dictionary
pi = provider_info.get(provider, {})
return render_template("show_ra.html",
pi=pi,
provider=provider,
providers=region_map.keys(),
provider_service_count=provider_service_count,
tlds=tlds,
provider_dataset_count=provider_dataset_count)
def inventory():
# provider list
providers = sorted(region_map.keys())
# service counts by provider
counts_by_provider = db.Service.count_types_by_provider_flat()
dataset_counts_by_provider = db.Dataset.count_types_by_provider_flat()
# get list of most recent updates since yesterday
since = datetime.utcnow() - timedelta(hours=24)
stats = list(
db.PingLatest.find({
'updated': {
'$gte': since
}
}).sort([('updated', -1)]))
upd_datasets = list(
db.Dataset.find({
'updated': {
'$gte': since
}
}).sort([('updated', -1)]))
services = db.Service.find({'_id': {'$in': [p.service_id for p in stats]}})
services = {s._id: s for s in services}
updates = []
for s in stats:
try:
updates.append({
'data_provider': services[s._id].data_provider,
'name': services[s._id].name,
'service_type': services[s._id].service_type,
'update_type': 'ping',
'updated': int(s.updated.strftime('%s')),
'updated_display': prettydate(s.updated),
'data': {
'code': s.last_response_code,
'time': s.last_response_time
},
'_id': str(s._id),
'url': url_for('show_service', service_id=s._id)
})
except KeyError:
app.logger.exception('KeyError updating stats')
for d in upd_datasets:
for s in d.services:
# due to the grouping, we may have older ones in this dataset
if s['updated'] != d.updated:
continue
updates.append({
'data_provider': s['data_provider'],
'name': s['name'],
'service_type': s['service_type'],
'update_type': 'harvest',
'updated': int(d.updated.strftime('%s')),
'updated_display': prettydate(d.updated),
'data': {},
'id': str(d._id),
'url': url_for('show_dataset', dataset_id=d._id)
})
return render_template(
'inventory.html',
counts_by_provider=counts_by_provider,
dataset_counts_by_provider=dataset_counts_by_provider,
updates=updates,
providers=providers)
