def get(self, portalid, snapshot):
session = current_app.config['dbsession']
prevWeek = getPreviousWeek(snapshot)
ds_cur = {
r.id: r.md5
for r in session.query(Dataset).filter(
Dataset.portalid == portalid).filter(
Dataset.snapshot == snapshot)
}
ds_prev = {
r.id: r.md5
for r in session.query(Dataset).filter(
Dataset.portalid == portalid).filter(
Dataset.snapshot == prevWeek)
}
data = {'adds': [], 'dels': [], 'updates': []}
for id, md5 in ds_cur.items():
if id not in ds_prev:
name = get_ds_name(session, md5, id)
data['adds'].append((id, name))
else:
if md5 != ds_prev[id]:
name = get_ds_name(session, ds_prev[id], id)
data['updates'].append((id, name))
# del key from prev snapshot
del ds_prev[id]
for id, md5 in ds_prev.items():
if id not in ds_cur:
name = get_ds_name(session, md5, id)
data['dels'].append((id, name))
return jsonify(data)
def portal(portalid, snapshot=getSnapshotfromTime(datetime.datetime.now())):
with Timer(key="get_portal", verbose=True):
current_sn = snapshot
Session = current_app.config['dbsession']
data = getPortalInfos(Session, portalid, snapshot)
dynamicityEnabled = current_app.config.get('dynamicity', False)
with Timer(key="query_portal", verbose=True):
q = Session.query(Portal).filter(Portal.id == portalid) \
.join(PortalSnapshotQuality, PortalSnapshotQuality.portalid == Portal.id) \
.filter(PortalSnapshotQuality.snapshot == snapshot) \
.join(PortalSnapshot, PortalSnapshot.portalid == Portal.id) \
.filter(PortalSnapshot.snapshot == snapshot) \
.add_entity(PortalSnapshot) \
.add_entity(PortalSnapshotQuality)
if dynamicityEnabled:
q = q.join(PortalSnapshotDynamicity, PortalSnapshotDynamicity.portalid == Portal.id) \
.filter(PortalSnapshotDynamicity.snapshot == snapshot) \
.add_entity(PortalSnapshotDynamicity)
r = q.first()
while r is None:
snapshot = getPreviousWeek(snapshot)
q = Session.query(Portal).filter(Portal.id == portalid) \
.join(PortalSnapshotQuality, PortalSnapshotQuality.portalid == Portal.id) \
.filter(PortalSnapshotQuality.snapshot == snapshot) \
.join(PortalSnapshot, PortalSnapshot.portalid == Portal.id) \
.filter(PortalSnapshot.snapshot == snapshot) \
.add_entity(PortalSnapshot) \
.add_entity(PortalSnapshotQuality)
if dynamicityEnabled:
q = q.join(PortalSnapshotDynamicity, PortalSnapshotDynamicity.portalid == Portal.id) \
.filter(PortalSnapshotDynamicity.snapshot == snapshot) \
.add_entity(PortalSnapshotDynamicity)
r = q.first()
data['portal'] = row2dict(r[0])
data['fetchInfo'] = row2dict(r[1])
data['fetchInfo']['duration'] = data['fetchInfo']['end'] - data[
'fetchInfo']['start']
if dynamicityEnabled:
data['dynamicity'] = row2dict(r[3])
data['quality'] = row2dict(r[2])
#with Timer(key="query_portal_agg", verbose=True):
# data.update(aggregatePortalInfo(Session,portalid,snapshot,dbc))
return render("odpw_portal.jinja",
snapshot=current_sn,
portalid=portalid,
data=data)
def getPortalInfos(Session, portalid, snapshot):
with Timer(key="get_getPortalInfos", verbose=True):
with Timer(key="query_getPortalInfos", verbose=True):
snapshots = [
i[0] for i in Session.query(PortalSnapshot.snapshot).filter(
PortalSnapshot.portalid == portalid).all()
]
p = getPreviousWeek(snapshot)
p = p if p in snapshots else None
n = getNextWeek(snapshot)
n = n if n in snapshots else None
data = {'snapshots': {'list': snapshots, 'prev': p, 'next': n}}
return data
def systemchanges():
with Timer(key="get_systemchanges"):
Session = current_app.config['dbsession']
cursn = getSnapshotfromTime(datetime.datetime.now())
prevWeek = getPreviousWeek(cursn)
with Timer(key="query_systemchanges"):
data_cur = {
r.portalid: r
for r in Session.query(PortalSnapshot).filter(
PortalSnapshot.snapshot == cursn)
}
data_prev = {
r.portalid: r
for r in Session.query(PortalSnapshot).filter(
PortalSnapshot.snapshot == prevWeek)
}
data = {'status_change': {}, 'ds_change': {}, 'res_change': {}}
for pid, ps in data_cur.items():
if pid in data_prev:
if ps.status == data_prev[pid].status:
if ps.datasetcount != data_prev[pid].datasetcount:
dsfrom = data_prev[pid].datasetcount if data_prev[
pid].datasetcount is not None else 0
dsto = ps.datasetcount if ps.datasetcount is not None else 0
data['ds_change'][pid] = {'from': dsfrom, 'to': dsto}
elif ps.resourcecount != data_prev[pid].resourcecount:
resfrom = data_prev[pid].resourcecount if data_prev[
pid].resourcecount is not None else 0
resto = ps.resourcecount if ps.resourcecount is not None else 0
data['res_change'][pid] = {
'from': resfrom,
'to': resto
}
else:
data['status_change'][pid] = {
'from': data_prev[pid].status,
'to': ps.status
}
data['from'] = prevWeek
data['to'] = cursn
return render("odpw_system_changes.jinja", data=data)
def licensesearch(uri=None):
with Timer(key="get_licensesearch", verbose=True):
data = {}
if uri != None:
cursn = getPreviousWeek(
getSnapshotfromTime(datetime.datetime.now()))
Session = current_app.config['dbsession']
with Timer(key="query_licensesearch"):
q = Session.query(Dataset, DatasetData) \
.join(MetaResource, Dataset.md5 == MetaResource.md5) \
.join(DatasetData, Dataset.md5 == DatasetData.md5) \
.filter(Dataset.snapshot == cursn) \
.filter(MetaResource.uri == uri)
results = []
for r in q:
results.append(row2dict(r))
data['uri'] = uri
data['snapshot'] = cursn
data['results'] = results
return render("odpw_license_search.jinja", data=data)