def get(self, portalid, snapshot): session = current_app.config['dbsession'] prevWeek = getPreviousWeek(snapshot) ds_cur = { r.id: r.md5 for r in session.query(Dataset).filter( Dataset.portalid == portalid).filter( Dataset.snapshot == snapshot) } ds_prev = { r.id: r.md5 for r in session.query(Dataset).filter( Dataset.portalid == portalid).filter( Dataset.snapshot == prevWeek) } data = {'adds': [], 'dels': [], 'updates': []} for id, md5 in ds_cur.items(): if id not in ds_prev: name = get_ds_name(session, md5, id) data['adds'].append((id, name)) else: if md5 != ds_prev[id]: name = get_ds_name(session, ds_prev[id], id) data['updates'].append((id, name)) # del key from prev snapshot del ds_prev[id] for id, md5 in ds_prev.items(): if id not in ds_cur: name = get_ds_name(session, md5, id) data['dels'].append((id, name)) return jsonify(data)
def portal(portalid, snapshot=getSnapshotfromTime(datetime.datetime.now())): with Timer(key="get_portal", verbose=True): current_sn = snapshot Session = current_app.config['dbsession'] data = getPortalInfos(Session, portalid, snapshot) dynamicityEnabled = current_app.config.get('dynamicity', False) with Timer(key="query_portal", verbose=True): q = Session.query(Portal).filter(Portal.id == portalid) \ .join(PortalSnapshotQuality, PortalSnapshotQuality.portalid == Portal.id) \ .filter(PortalSnapshotQuality.snapshot == snapshot) \ .join(PortalSnapshot, PortalSnapshot.portalid == Portal.id) \ .filter(PortalSnapshot.snapshot == snapshot) \ .add_entity(PortalSnapshot) \ .add_entity(PortalSnapshotQuality) if dynamicityEnabled: q = q.join(PortalSnapshotDynamicity, PortalSnapshotDynamicity.portalid == Portal.id) \ .filter(PortalSnapshotDynamicity.snapshot == snapshot) \ .add_entity(PortalSnapshotDynamicity) r = q.first() while r is None: snapshot = getPreviousWeek(snapshot) q = Session.query(Portal).filter(Portal.id == portalid) \ .join(PortalSnapshotQuality, PortalSnapshotQuality.portalid == Portal.id) \ .filter(PortalSnapshotQuality.snapshot == snapshot) \ .join(PortalSnapshot, PortalSnapshot.portalid == Portal.id) \ .filter(PortalSnapshot.snapshot == snapshot) \ .add_entity(PortalSnapshot) \ .add_entity(PortalSnapshotQuality) if dynamicityEnabled: q = q.join(PortalSnapshotDynamicity, PortalSnapshotDynamicity.portalid == Portal.id) \ .filter(PortalSnapshotDynamicity.snapshot == snapshot) \ .add_entity(PortalSnapshotDynamicity) r = q.first() data['portal'] = row2dict(r[0]) data['fetchInfo'] = row2dict(r[1]) data['fetchInfo']['duration'] = data['fetchInfo']['end'] - data[ 'fetchInfo']['start'] if dynamicityEnabled: data['dynamicity'] = row2dict(r[3]) data['quality'] = row2dict(r[2]) #with Timer(key="query_portal_agg", verbose=True): # data.update(aggregatePortalInfo(Session,portalid,snapshot,dbc)) return render("odpw_portal.jinja", snapshot=current_sn, portalid=portalid, data=data)
def getPortalInfos(Session, portalid, snapshot): with Timer(key="get_getPortalInfos", verbose=True): with Timer(key="query_getPortalInfos", verbose=True): snapshots = [ i[0] for i in Session.query(PortalSnapshot.snapshot).filter( PortalSnapshot.portalid == portalid).all() ] p = getPreviousWeek(snapshot) p = p if p in snapshots else None n = getNextWeek(snapshot) n = n if n in snapshots else None data = {'snapshots': {'list': snapshots, 'prev': p, 'next': n}} return data
def systemchanges(): with Timer(key="get_systemchanges"): Session = current_app.config['dbsession'] cursn = getSnapshotfromTime(datetime.datetime.now()) prevWeek = getPreviousWeek(cursn) with Timer(key="query_systemchanges"): data_cur = { r.portalid: r for r in Session.query(PortalSnapshot).filter( PortalSnapshot.snapshot == cursn) } data_prev = { r.portalid: r for r in Session.query(PortalSnapshot).filter( PortalSnapshot.snapshot == prevWeek) } data = {'status_change': {}, 'ds_change': {}, 'res_change': {}} for pid, ps in data_cur.items(): if pid in data_prev: if ps.status == data_prev[pid].status: if ps.datasetcount != data_prev[pid].datasetcount: dsfrom = data_prev[pid].datasetcount if data_prev[ pid].datasetcount is not None else 0 dsto = ps.datasetcount if ps.datasetcount is not None else 0 data['ds_change'][pid] = {'from': dsfrom, 'to': dsto} elif ps.resourcecount != data_prev[pid].resourcecount: resfrom = data_prev[pid].resourcecount if data_prev[ pid].resourcecount is not None else 0 resto = ps.resourcecount if ps.resourcecount is not None else 0 data['res_change'][pid] = { 'from': resfrom, 'to': resto } else: data['status_change'][pid] = { 'from': data_prev[pid].status, 'to': ps.status } data['from'] = prevWeek data['to'] = cursn return render("odpw_system_changes.jinja", data=data)
def licensesearch(uri=None): with Timer(key="get_licensesearch", verbose=True): data = {} if uri != None: cursn = getPreviousWeek( getSnapshotfromTime(datetime.datetime.now())) Session = current_app.config['dbsession'] with Timer(key="query_licensesearch"): q = Session.query(Dataset, DatasetData) \ .join(MetaResource, Dataset.md5 == MetaResource.md5) \ .join(DatasetData, Dataset.md5 == DatasetData.md5) \ .filter(Dataset.snapshot == cursn) \ .filter(MetaResource.uri == uri) results = [] for r in q: results.append(row2dict(r)) data['uri'] = uri data['snapshot'] = cursn data['results'] = results return render("odpw_license_search.jinja", data=data)