Exemple #1
0
def show_ra(provider):
    if not provider in region_map:
        return redirect('/')

    # get numbers for the RA
    service_counts = db.Service.count_types_by_provider()
    dataset_counts = db.Dataset.count_types_by_provider()

    provider_service_count = service_counts.get(provider, {}).get('_all', 0)
    provider_dataset_count = dataset_counts.get(provider, {}).get('_all', 0)

    # get tls for this RA
    tlds = db.Service.find({
        'data_provider': provider,
        'active': True
    }).distinct('tld')

    # get provider extra info from dictionary
    pi = provider_info.get(provider, {})

    return render_template("show_ra.html",
                           pi=pi,
                           provider=provider,
                           providers=region_map.keys(),
                           provider_service_count=provider_service_count,
                           tlds=tlds,
                           provider_dataset_count=provider_dataset_count)
Exemple #2
0
def view_metadatas(filter_provider):
    service_filters = {'active': True}
    if filter_provider is not None:
        service_filters['data_provider'] = filter_provider

    sids = get_service_ids(service_filters)
    metadatas, cols, dids = get_metadatas(sids)

    # get mappings of services/datasets
    services = {
        s._id: s
        for s in db.Service.find({'_id': {
            '$in': list(sids)
        }})
    }
    #datasets = {d._id:d.name for d in db.Dataset.find({'_id':{'$in':list(dids)}})}

    return render_template(
        "metadatas.html",
        metadatas=metadatas,
        services=services,
        #datasets=datasets,
        providers=region_map.keys(),
        filters=service_filters,
        columns=list(cols))
Exemple #3
0
def view_metadatas(filter_provider):
    service_filters = {'active': True}
    if filter_provider is not None:
        service_filters['data_provider'] = filter_provider

    sids = get_service_ids(service_filters)
    metadatas, cols, dids = get_metadatas(sids)

    # get mappings of services/datasets
    services = {
        s._id: s
        for s in db.Service.find({
            '_id': {
                '$in': list(sids)
            }
        })
    }
    #datasets = {d._id:d.name for d in db.Dataset.find({'_id':{'$in':list(dids)}})}

    return render_template(
        "metadatas.html",
        metadatas=metadatas,
        services=services,
        #datasets=datasets,
        providers=region_map.keys(),
        filters=service_filters,
        columns=list(cols))
Exemple #4
0
def dashboard():
    # provider list
    providers = sorted(region_map.keys())

    # service counts by provider
    counts_by_provider = db.Service.count_types_by_provider_flat()
    dataset_counts_by_provider = db.Dataset.count_types_by_provider_flat()

    # get list of most recent updates since yesterday
    since        = datetime.utcnow() - timedelta(hours=24)

    stats        = list(db.PingLatest.find({'updated':{'$gte':since}}).sort([('updated',-1)]))
    upd_datasets = list(db.Dataset.find({'updated':{'$gte':since}}).sort([('updated', -1)]))
    services     = db.Service.find({'_id':{'$in':[p.service_id for p in stats]}})
    services     = {s._id:s for s in services}

    updates = []

    for s in stats:
        try:
            updates.append({'data_provider':services[s._id].data_provider,
                            'name': services[s._id].name,
                            'service_type': services[s._id].service_type,
                            'update_type': 'ping',
                            'updated': int(s.updated.strftime('%s')),
                            'updated_display': prettydate(s.updated),
                            'data': {'code':s.last_response_code,
                                     'time':s.last_response_time},
                            '_id': str(s._id),
                            'url': url_for('show_service', service_id=s._id)})
        except KeyError:
            app.logger.exception('KeyError updating stats')

    for d in upd_datasets:
        for s in d.services:
            # due to the grouping, we may have older ones in this dataset
            if s['updated'] != d.updated:
                continue

            updates.append({'data_provider':s['data_provider'],
                            'name':s['name'],
                            'service_type': s['service_type'],
                            'update_type':'harvest',
                            'updated': int(d.updated.strftime('%s')),
                            'updated_display': prettydate(d.updated),
                            'data':{},
                            'id':str(d._id),
                            'url':url_for('show_dataset', dataset_id=d._id)})

    return render_template('index.html',
                           counts_by_provider=counts_by_provider,
                           dataset_counts_by_provider=dataset_counts_by_provider,
                           updates=updates,
                           providers=providers)
Exemple #5
0
def show_ra(provider):
    if not provider in region_map:
        return redirect('/')

    # get numbers for the RA
    service_counts = db.Service.count_types_by_provider()
    dataset_counts = db.Dataset.count_types_by_provider()

    provider_service_count = service_counts.get(provider, {}).get('_all', 0)
    provider_dataset_count = dataset_counts.get(provider, {}).get('_all', 0)

    # get tls for this RA
    tlds = db.Service.find({'data_provider':provider, 'active':True}).distinct('tld')

    # get provider extra info from dictionary
    pi = provider_info.get(provider, {})

    return render_template("show_ra.html",
                           pi=pi,
                           provider=provider,
                           providers=region_map.keys(),
                           provider_service_count=provider_service_count,
                           tlds=tlds,
                           provider_dataset_count=provider_dataset_count)
Exemple #6
0
def inventory():
    # provider list
    providers = sorted(region_map.keys())

    # service counts by provider
    counts_by_provider = db.Service.count_types_by_provider_flat()
    dataset_counts_by_provider = db.Dataset.count_types_by_provider_flat()

    # get list of most recent updates since yesterday
    since = datetime.utcnow() - timedelta(hours=24)

    stats = list(
        db.PingLatest.find({
            'updated': {
                '$gte': since
            }
        }).sort([('updated', -1)]))
    upd_datasets = list(
        db.Dataset.find({
            'updated': {
                '$gte': since
            }
        }).sort([('updated', -1)]))
    services = db.Service.find({'_id': {'$in': [p.service_id for p in stats]}})
    services = {s._id: s for s in services}

    updates = []

    for s in stats:
        try:
            updates.append({
                'data_provider': services[s._id].data_provider,
                'name': services[s._id].name,
                'service_type': services[s._id].service_type,
                'update_type': 'ping',
                'updated': int(s.updated.strftime('%s')),
                'updated_display': prettydate(s.updated),
                'data': {
                    'code': s.last_response_code,
                    'time': s.last_response_time
                },
                '_id': str(s._id),
                'url': url_for('show_service', service_id=s._id)
            })
        except KeyError:
            app.logger.exception('KeyError updating stats')

    for d in upd_datasets:
        for s in d.services:
            # due to the grouping, we may have older ones in this dataset
            if s['updated'] != d.updated:
                continue

            updates.append({
                'data_provider': s['data_provider'],
                'name': s['name'],
                'service_type': s['service_type'],
                'update_type': 'harvest',
                'updated': int(d.updated.strftime('%s')),
                'updated_display': prettydate(d.updated),
                'data': {},
                'id': str(d._id),
                'url': url_for('show_dataset', dataset_id=d._id)
            })

    return render_template(
        'inventory.html',
        counts_by_provider=counts_by_provider,
        dataset_counts_by_provider=dataset_counts_by_provider,
        updates=updates,
        providers=providers)