Пример #1
0
def portalResBody(snapshot, portalid):
    with Timer(key="get_portalRes", verbose=True):
        Session = current_app.config['dbsession']
        dbc = current_app.config['dbc']
        with Timer(key="query_portalRes", verbose=True):
            data = getResourceInfo(Session, dbc, portalid, snapshot)

        with Timer(key="query_getMetaResource", verbose=True):
            viewName = "view_{}_{}_{}".format('resinfo', portalid, snapshot)
            qorg = getResourceInfos(Session, snapshot, portalid)
            q = withView(qorg, viewName, Session, dbc)
            #print q
            start = time.time()
            data['uris'] = [row2dict(i) for i in q]
            end = time.time()
            if (end - start) > 5:
                print("Create View {}".format(viewName))
                createView(qorg, viewName, Session)

        data.update(getPortalInfos(Session, portalid, snapshot))
        #data['portals']= [ row2dict(r) for r in Session.query(Portal).all()]
        with Timer(key="query_portalResourceCount", verbose=True):
            r = current_app.config['dbsession'].query(
                Portal.resourcecount).filter(Portal.id == portalid)
            ps = []
            for P in r:
                data['resources'] = P[0]

        return render("odpw_portal_resources_list.jinja",
                      data=data,
                      snapshot=snapshot,
                      portalid=portalid)
Пример #2
0
def getResourceInfo(session, dbc, portalid, snapshot, orga=None):
    with Timer(key="getResourceInfo", verbose=True):
        data = {}

        with Timer(key="query_getResourceInfoValid", verbose=True):
            data['valid'] = {}
            for valid in validURLDist(session,
                                      snapshot,
                                      portalid=portalid,
                                      orga=orga):
                data['valid'][valid[0]] = valid[1]
        with Timer(key="query_getResourceInfoStatus", verbose=True):
            data['status'] = {}
            if not orga:
                viewName = "view_{}_{}_{}".format('resstatus', portalid,
                                                  snapshot)
            else:
                viewName = "view_{}_{}_{}_{}".format('resstatus', portalid,
                                                     snapshot, orga)

            qorg = statusCodeDist(session,
                                  snapshot,
                                  portalid=portalid,
                                  orga=orga)
            q = withView(qorg, viewName, session, dbc)
            start = time.time()
            for res in q:
                data['status'][res[0]] = res[1]
            end = time.time()
            if (end - start) > 5:
                print("Create View {}".format(viewName))
                createView(qorg, viewName, session)

        return {'resourcesInfo': data}
Пример #3
0
def aggregatePortalInfo(session, portalid, snapshot, dbc, limit=3):
    stats = {}
    with Timer(key=portalid + '-agg', verbose=True):
        ds = session.query(Dataset).filter(
            Dataset.snapshot == snapshot).filter(
                Dataset.portalid == portalid).count()
        rs = session.query(PortalSnapshot.resourcecount).filter(
            PortalSnapshot.portalid == portalid).filter(
                PortalSnapshot.snapshot == snapshot).first()
        print
        #print 'dsCount', ds
        #TODO fix resource count
        for key, cFunc, dFunc in [('organisation', organisationDist,
                                   distinctOrganisations),
                                  ('license', licenseDist, distinctLicenses),
                                  ('format', formatDist, distinctFormats)]:
            if key == 'format':
                total = row2dict(rs)['resourcecount']
            else:
                total = ds
            with Timer(key='query_{}-{}'.format(portalid, key), verbose=True):

                s = []

                viewName = "view_{}_{}_{}".format(key, portalid, snapshot)
                qorg = cFunc(session, snapshot, portalid=portalid)
                q = withView(qorg, viewName, session, dbc)
                start = time.time()
                if limit:
                    q = q.limit(limit)
                else:
                    q = q.all()
                for i in q:
                    d = row2dict(i)
                    #print d
                    d['perc'] = d['count'] / (1.0 * total) if total > 0 else 0
                    s.append(d)
                t = sum(item['count'] for item in s)
                #print key, 'total',t
                if ds - t != 0:
                    s.append({
                        key: 'Others',
                        'count': total - t,
                        'perc': (total - t) / (1.0 * total)
                    })
                end = time.time()
                if (end - start) > 5:
                    log.info("Create View {}".format(viewName))
                    createView(qorg, viewName, session)
                #q = withView(qorg, viewName, session, dbc)
                stats[key] = {
                    'distinct': dFunc(session, snapshot,
                                      portalid=portalid).count(),
                    'top3Dist': s
                }

    return stats
Пример #4
0
def resourceInfo(snapshot, portalid, uri):
    with Timer(key="get_resourceInfo", verbose=True):
        #print snapshot,portalid,uri

        Session = current_app.config['dbsession']
        dbc = current_app.config['dbc']
        data = getPortalInfos(Session, portalid, snapshot)

        with Timer(key="query_resources", verbose=True):
            viewName = "view_{}_{}_{}".format('resinfo', portalid, snapshot)
            qorg = getResourceInfos(Session, snapshot, portalid)
            q = withView(qorg, viewName, Session, dbc)
            start = time.time()
            data['resources'] = [row2dict(r) for r in q.all()]
            end = time.time()
            if (end - start) > 5:
                print("Create View {}".format(viewName))
                try:
                    createView(qorg, viewName, Session)
                except Exception as e:
                    if 'already exists' in e.message:
                        pass
                    else:
                        raise e

        with Timer(key="query_resourceInfo", verbose=True):
            q = Session.query(ResourceInfo) \
                .filter(ResourceInfo.uri == uri)
            #print q
            data['resourceInfo'] = [row2dict(r) for r in q.all()]

            for r in data['resourceInfo']:
                if 'header' in r:
                    if r['header'] is None:
                        r['header'] = ""
                    else:
                        #print type(r['header']),r['header'],r
                        r['header'] = ast.literal_eval(str(r['header']))

        return render("odpw_portal_resource.jinja",
                      snapshot=snapshot,
                      portalid=portalid,
                      uri=uri,
                      data=data)