def portalResBody(snapshot, portalid):
with Timer(key="get_portalRes", verbose=True):
Session = current_app.config['dbsession']
dbc = current_app.config['dbc']
with Timer(key="query_portalRes", verbose=True):
data = getResourceInfo(Session, dbc, portalid, snapshot)
with Timer(key="query_getMetaResource", verbose=True):
viewName = "view_{}_{}_{}".format('resinfo', portalid, snapshot)
qorg = getResourceInfos(Session, snapshot, portalid)
q = withView(qorg, viewName, Session, dbc)
#print q
start = time.time()
data['uris'] = [row2dict(i) for i in q]
end = time.time()
if (end - start) > 5:
print("Create View {}".format(viewName))
createView(qorg, viewName, Session)
data.update(getPortalInfos(Session, portalid, snapshot))
#data['portals']= [ row2dict(r) for r in Session.query(Portal).all()]
with Timer(key="query_portalResourceCount", verbose=True):
r = current_app.config['dbsession'].query(
Portal.resourcecount).filter(Portal.id == portalid)
ps = []
for P in r:
data['resources'] = P[0]
return render("odpw_portal_resources_list.jinja",
data=data,
snapshot=snapshot,
portalid=portalid)
def getResourceInfo(session, dbc, portalid, snapshot, orga=None):
with Timer(key="getResourceInfo", verbose=True):
data = {}
with Timer(key="query_getResourceInfoValid", verbose=True):
data['valid'] = {}
for valid in validURLDist(session,
snapshot,
portalid=portalid,
orga=orga):
data['valid'][valid[0]] = valid[1]
with Timer(key="query_getResourceInfoStatus", verbose=True):
data['status'] = {}
if not orga:
viewName = "view_{}_{}_{}".format('resstatus', portalid,
snapshot)
else:
viewName = "view_{}_{}_{}_{}".format('resstatus', portalid,
snapshot, orga)
qorg = statusCodeDist(session,
snapshot,
portalid=portalid,
orga=orga)
q = withView(qorg, viewName, session, dbc)
start = time.time()
for res in q:
data['status'][res[0]] = res[1]
end = time.time()
if (end - start) > 5:
print("Create View {}".format(viewName))
createView(qorg, viewName, session)
return {'resourcesInfo': data}
def aggregatePortalInfo(session, portalid, snapshot, dbc, limit=3):
stats = {}
with Timer(key=portalid + '-agg', verbose=True):
ds = session.query(Dataset).filter(
Dataset.snapshot == snapshot).filter(
Dataset.portalid == portalid).count()
rs = session.query(PortalSnapshot.resourcecount).filter(
PortalSnapshot.portalid == portalid).filter(
PortalSnapshot.snapshot == snapshot).first()
print
#print 'dsCount', ds
#TODO fix resource count
for key, cFunc, dFunc in [('organisation', organisationDist,
distinctOrganisations),
('license', licenseDist, distinctLicenses),
('format', formatDist, distinctFormats)]:
if key == 'format':
total = row2dict(rs)['resourcecount']
else:
total = ds
with Timer(key='query_{}-{}'.format(portalid, key), verbose=True):
s = []
viewName = "view_{}_{}_{}".format(key, portalid, snapshot)
qorg = cFunc(session, snapshot, portalid=portalid)
q = withView(qorg, viewName, session, dbc)
start = time.time()
if limit:
q = q.limit(limit)
else:
q = q.all()
for i in q:
d = row2dict(i)
#print d
d['perc'] = d['count'] / (1.0 * total) if total > 0 else 0
s.append(d)
t = sum(item['count'] for item in s)
#print key, 'total',t
if ds - t != 0:
s.append({
key: 'Others',
'count': total - t,
'perc': (total - t) / (1.0 * total)
})
end = time.time()
if (end - start) > 5:
log.info("Create View {}".format(viewName))
createView(qorg, viewName, session)
#q = withView(qorg, viewName, session, dbc)
stats[key] = {
'distinct': dFunc(session, snapshot,
portalid=portalid).count(),
'top3Dist': s
}
return stats
def resourceInfo(snapshot, portalid, uri):
with Timer(key="get_resourceInfo", verbose=True):
#print snapshot,portalid,uri
Session = current_app.config['dbsession']
dbc = current_app.config['dbc']
data = getPortalInfos(Session, portalid, snapshot)
with Timer(key="query_resources", verbose=True):
viewName = "view_{}_{}_{}".format('resinfo', portalid, snapshot)
qorg = getResourceInfos(Session, snapshot, portalid)
q = withView(qorg, viewName, Session, dbc)
start = time.time()
data['resources'] = [row2dict(r) for r in q.all()]
end = time.time()
if (end - start) > 5:
print("Create View {}".format(viewName))
try:
createView(qorg, viewName, Session)
except Exception as e:
if 'already exists' in e.message:
pass
else:
raise e
with Timer(key="query_resourceInfo", verbose=True):
q = Session.query(ResourceInfo) \
.filter(ResourceInfo.uri == uri)
#print q
data['resourceInfo'] = [row2dict(r) for r in q.all()]
for r in data['resourceInfo']:
if 'header' in r:
if r['header'] is None:
r['header'] = ""
else:
#print type(r['header']),r['header'],r
r['header'] = ast.literal_eval(str(r['header']))
return render("odpw_portal_resource.jinja",
snapshot=snapshot,
portalid=portalid,
uri=uri,
data=data)