def index():
collections = authz.collections(authz.READ)
if not len(collections):
return jsonify(Pager([]))
q = session.query(Source)
q = q.filter(Source.id.in_(collections))
q = q.order_by(Source.title.asc())
return jsonify(Pager(q))
def index():
collections = authz.collections(authz.READ)
if not len(collections):
return jsonify(Pager([]))
q = session.query(Collection)
q = q.filter(Collection.id.in_(collections))
q = q.options(subqueryload('subjects'))
q = q.order_by(Collection.title.asc())
return jsonify(Pager(q))
def match():
dataset_arg = request.args.get('dataset')
dataset = Dataset.find(dataset_arg)
matches = find_matches(dataset,
request.args.get('name'),
filter=request.args.get('filter'),
exclude=arg_int('exclude'))
pager = Pager(matches)
return jsonify(pager.to_dict())
def match():
dataset_arg = request.args.get('dataset')
dataset = Dataset.find(dataset_arg)
matches = find_matches(dataset,
request.args.get('name'),
filter=request.args.get('filter'),
exclude=arg_int('exclude'))
pager = Pager(matches)
return jsonify(pager.to_dict())
def index():
q = List.all_by_user(current_user)
data = Pager(q).to_dict()
results = []
for lst in data.pop('results'):
ldata = lst.to_dict()
ldata['can_write'] = authz.list_write(lst.id)
results.append(ldata)
data['results'] = results
return jsonify(data)
def index():
q = Collection.all_by_user(current_user)
data = Pager(q).to_dict()
results = []
for lst in data.pop('results'):
ldata = lst.to_dict()
ldata['permissions'] = {'write': authz.collection_write(lst.id)}
results.append(ldata)
data['results'] = results
return jsonify(data)
def query():
etag_cache_keygen()
query = document_query(request.args, lists=authz.authz_lists('read'),
sources=authz.authz_sources('read'))
results = search_documents(query)
pager = Pager(results,
results_converter=lambda ds: [add_urls(d) for d in ds])
data = pager.to_dict()
data['facets'] = transform_facets(results.result.get('aggregations', {}))
return jsonify(data)
def index():
q = Collection.all_by_user(current_user)
data = Pager(q).to_dict()
results = []
for lst in data.pop('results'):
ldata = lst.to_dict()
ldata['permissions'] = {
'write': authz.collection_write(lst.id)
}
results.append(ldata)
data['results'] = results
return jsonify(data)
def load_relations(entity, id, slug):
relation_sections = []
q = grano.relations.query().limit(0)
q = q.filter('facet', 'schema').filter('entity', id)
schema_types = q.data.get('facets', {}).get('schema', {})
for (schema, count) in schema_types.get('results', []):
iq = grano.relations.query().limit(15)
iq = iq.filter('entity', id)
iq = iq.filter('schema', schema.get('name'))
pager = Pager(iq, name=schema.get('name'), id=id, slug=slug)
relations = []
for r in pager:
r.props = Properties(r)
r.other = r.source if r.target.id == id else r.target
relations.append(r)
data = {
'schema': schemata.by_name(schema.get('name')),
'count': count,
'pager': pager,
'relations': relations
}
relation_sections.append(data)
return sorted(relation_sections, key=lambda r: r['schema'].label)
def index(dataset):
dataset = get_dataset(dataset)
q = Run.all(dataset)
if 'source' in request.args:
q = q.filter(Run.source == request.args.get('source'))
pager = Pager(q, dataset=dataset.name)
return jsonify(pager)
def index():
sources_ids = match_ids('sources', authz.sources(authz.READ))
q = Document.all().filter(Document.source_id.in_(sources_ids))
hashes = request.args.getlist('content_hash')
if len(hashes):
q = q.filter(Document.content_hash.in_(hashes))
return jsonify(Pager(q))
def query_index():
q = Dataset.all_by_account(current_user, order=False)
q = q.order_by(Dataset.updated_at.desc())
# Filter by languages if they have been provided
for language in request.args.getlist('languages'):
l = aliased(DatasetLanguage)
q = q.join(l, Dataset._languages)
q = q.filter(l.code == language)
# Filter by territories if they have been provided
for territory in request.args.getlist('territories'):
t = aliased(DatasetTerritory)
q = q.join(t, Dataset._territories)
q = q.filter(t.code == territory)
# Filter by account if one has been provided
for account in request.args.getlist('account'):
a = aliased(Account)
q = q.join(a, Dataset.managers)
q = q.filter(a.name == account)
# Return a list of languages as dicts with code, count, url and label
languages = [{'code': code, 'count': count, 'label': LANGUAGES.get(code)}
for (code, count) in DatasetLanguage.dataset_counts(q)]
territories = [{'code': code, 'count': count, 'label': COUNTRIES.get(code)}
for (code, count) in DatasetTerritory.dataset_counts(q)]
pager = Pager(q, limit=15)
return pager, languages, territories
def all():
q = Entity.all()
q = q.filter(Entity.state == Entity.STATE_ACTIVE)
clause = Collection.id.in_(authz.collections(authz.READ))
q = q.filter(Entity.collections.any(clause))
q = q.order_by(Entity.id.asc())
return jsonify(Pager(q, limit=100))
def _query():
'''
everything here should be applicable both to the internal and to the
public api
'''
etag_cache_keygen()
query = document_query(request.args, lists=authz.authz_lists('read'),
sources=authz.authz_sources('read'),
highlights=True)
results = search_documents(query)
pager = Pager(results,
results_converter=lambda ds: [add_urls(d) for d in ds])
data = pager.to_dict()
#import ipdb; ipdb.set_trace()
data['facets'] = transform_facets(results.result.get('aggregations', {}))
return data
def _query():
'''
everything here should be applicable both to the internal and to the
public api
'''
etag_cache_keygen()
query = document_query(request.args,
lists=authz.authz_lists('read'),
sources=authz.authz_sources('read'),
highlights=True)
results = search_documents(query)
pager = Pager(results,
results_converter=lambda ds: [add_urls(d) for d in ds])
data = pager.to_dict()
#import ipdb; ipdb.set_trace()
data['facets'] = transform_facets(results.result.get('aggregations', {}))
return data
def index():
collection_ids = match_ids('collection', authz.collections(authz.READ))
q = Document.all()
clause = Collection.id.in_(collection_ids)
q = q.filter(Document.collections.any(clause))
hashes = request.args.getlist('content_hash')
if len(hashes):
q = q.filter(Document.content_hash.in_(hashes))
return jsonify(Pager(q))
def index():
authz = request.authz
collections = request.args.getlist('collection')
collections = authz.collections_intersect(authz.READ, collections)
q = Document.all()
q = q.filter(Document.collection_id.in_(collections))
hashes = request.args.getlist('content_hash')
if len(hashes):
q = q.filter(Document.content_hash.in_(hashes))
return jsonify(Pager(q))
def index(dataset):
dataset = get_dataset(dataset)
package = data_manager.package(dataset.name)
sources = list(package.all(Source))
sources = sorted(sources,
key=lambda s: s.meta.get('updated_at'),
reverse=True)
rc = lambda ss: [source_to_dict(dataset, s) for s in ss]
return jsonify(
Pager(sources, dataset=dataset.name, limit=5, results_converter=rc))
def references(document_id):
doc = get_document(document_id)
q = db.session.query(Reference)
q = q.filter(Reference.document_id == doc.id)
q = q.filter(Reference.origin == 'regex')
q = q.join(Entity)
q = q.filter(Entity.state == Entity.STATE_ACTIVE)
q = q.filter(Entity.collection_id.in_(request.authz.collections_read))
q = q.order_by(Reference.weight.desc())
return jsonify(Pager(q, document_id=document_id))
def complete(format='json'):
if not current_user.is_authenticated():
msg = _("You are not authorized to see that page")
return jsonify({'status': 'error', 'message': msg}, status=403)
query = db.session.query(Account)
filter_string = request.args.get('q', '') + '%'
query = query.filter(or_(Account.name.ilike(filter_string),
Account.fullname.ilike(filter_string)))
return jsonify(Pager(query))
def states():
authz.require(authz.is_admin())
q = db.session.query(CrawlerState)
q = q.filter(CrawlerState.status == CrawlerState.STATUS_FAIL)
if 'crawler_id' in request.args:
q = q.filter(CrawlerState.crawler_id == request.args.get('crawler_id'))
if 'crawler_run' in request.args:
q = q.filter(CrawlerState.crawler_run == request.args.get('crawler_run'))
if 'error_type' in request.args:
q = q.filter(CrawlerState.error_type == request.args.get('error_type'))
q = q.order_by(CrawlerState.created_at.desc())
response = Pager(q).to_dict()
return jsonify(response)
def index():
list_ids = List.user_list_ids(current_user)
filter_lists = request.args.getlist('list')
if len(filter_lists):
try:
filter_lists = [int(f) for f in filter_lists]
list_ids = [l for l in list_ids if l in filter_lists]
except ValueError:
raise BadRequest()
prefix = request.args.get('prefix')
q = Entity.by_lists(list_ids, prefix=prefix)
return jsonify(Pager(q))
def index():
entities = Entity.all()
dataset_arg = request.args.get("dataset")
if dataset_arg is not None:
dataset = Dataset.find(dataset_arg)
entities = entities.filter_by(dataset=dataset)
filter_name = request.args.get("filter_name", "")
if len(filter_name):
query = "%" + filter_name + "%"
entities = entities.filter(Entity.name.ilike(query))
# TODO, other filters.
format = request.args.get("format", "json").lower().strip()
if format == "csv":
res = csvify(entities)
else:
pager = Pager(entities)
res = jsonify(pager.to_dict())
if arg_bool("download"):
fn = dataset_filename(dataset, format)
res.headers["Content-Disposition"] = "attachment; filename=" + fn
return res
def index():
entities = Entity.all()
dataset_arg = request.args.get('dataset')
if dataset_arg is not None:
dataset = Dataset.find(dataset_arg)
entities = entities.filter_by(dataset=dataset)
filter_name = request.args.get('filter_name', '')
if len(filter_name):
query = '%' + filter_name + '%'
entities = entities.filter(Entity.name.ilike(query))
# TODO, other filters.
format = request.args.get('format', 'json').lower().strip()
if format == 'csv':
res = csvify(entities)
else:
pager = Pager(entities)
res = jsonify(pager.to_dict())
if arg_bool('download'):
fn = dataset_filename(dataset, format)
res.headers['Content-Disposition'] = 'attachment; filename=' + fn
return res
def index():
collection_ids = Collection.user_ids(current_user)
filter_collections = request.args.getlist('collection')
if len(filter_collections):
try:
collection_ids = [
l for l in collection_ids if l in filter_collections
]
except ValueError:
raise BadRequest()
prefix = request.args.get('prefix')
q = Entity.by_collection(collection_ids, prefix=prefix)
return jsonify(Pager(q))
def suggest():
request.authz.require(request.authz.logged_in)
prefix = stringify(request.args.get('prefix'))
if prefix is None or len(prefix) < 3:
# Do not return 400 because it's a routine event.
return jsonify({
'status': 'error',
'message': 'prefix filter is too short',
'results': [],
'total': 0
})
# this only returns users, not groups
q = Role.by_prefix(prefix)
return jsonify(Pager(q, limit=10))
def index():
try:
authorized = authz.collections(authz.READ)
collection_ids = [int(f) for f in request.args.getlist('collection')]
collection_ids = collection_ids or authorized
collection_ids = [c for c in collection_ids if c in authorized]
except ValueError:
raise BadRequest()
q = Document.all()
clause = Collection.id.in_(collection_ids)
q = q.filter(Document.collections.any(clause))
hashes = request.args.getlist('content_hash')
if len(hashes):
q = q.filter(Document.content_hash.in_(hashes))
return jsonify(Pager(q))
def paths(id):
collection = obj_or_404(Collection.by_id(id))
authz.require(authz.collection_read(collection.id))
start_entity_id = request.args.get('entity_id')
labels = request.args.getlist('label')
types = request.args.getlist('type')
collection_id = request.args.getlist('collection_id')
end_collection_id = authz.collections_intersect(authz.READ, collection_id)
q = Path.find(collection, start_entity_id=start_entity_id, labels=labels,
types=types, end_collection_id=end_collection_id)
data = Pager(q, id=collection.id).to_dict()
data['facets'] = Path.facets(collection, start_entity_id=start_entity_id,
labels=labels, types=types,
end_collection_id=end_collection_id,
collection_id=authz.collections(authz.READ))
return jsonify(data)
def view():
query = grano.entities.query()
query = query.filter('q', request.args.get('q', ''))
schema_active = request.args.getlist('schema')
main_schema = None
if not len(schema_active):
# TODO: get this from schema 'meta':
schema_active = [
'Person', 'Organization', 'Company', 'NonProfit',
'EducationalInstitution'
]
else:
main_schema = schemata.by_name(schema_active[0])
query = query.filter('schema', schema_active)
query = query.filter('sort', '-degree')
pager = Pager(query)
return render_template('browser.html',
main_schema=main_schema,
pager=pager)
def index():
# allow to filter for writeable collections only, needed
# in some UI scenarios:
state = QueryState(request.args, request.authz)
permission = request.args.get('permission')
if permission not in [request.authz.READ, request.authz.WRITE]:
permission = request.authz.READ
collections = request.authz.collections[permission]
# Other filters for navigation
label = request.args.get('label')
managed = state.getbool('managed', None)
# Include counts (of entities, documents) in list view?
counts = state.getbool('counts', False)
def converter(colls):
return [c.to_dict(counts=counts) for c in colls]
facet = [f.lower().strip() for f in request.args.getlist('facet')]
q = Collection.find(label=label,
countries=state.getfilter('countries'),
category=state.getfilter('category'),
collection_id=collections,
managed=managed)
data = Pager(q).to_dict(results_converter=converter)
facets = {}
if 'countries' in facet:
facets['countries'] = {
'values':
Collection.facet_by(q, Collection.countries, mapping=COUNTRY_NAMES)
}
if 'category' in facet:
mapping = get_config('COLLECTION_CATEGORIES', {})
facets['category'] = {
'values': Collection.facet_by(q,
Collection.category,
mapping=mapping)
}
data['facets'] = facets
return jsonify(data)
def collection_crawlerstates(id):
request.authz.require(request.authz.collection_read(id))
q = db.session.query(CrawlerState)
q = q.filter(CrawlerState.collection_id == id)
q = q.filter(or_(
CrawlerState.error_type != 'init',
CrawlerState.error_type == None # noqa
))
status = request.args.get('status')
if status:
q = q.filter(CrawlerState.status == status)
crawler_id = request.args.get('crawler_id')
if crawler_id:
q = q.filter(CrawlerState.crawler_id == crawler_id)
crawler_run = request.args.get('crawler_run')
if crawler_run:
q = q.filter(CrawlerState.crawler_run == crawler_run)
q = q.order_by(CrawlerState.created_at.desc())
return jsonify(Pager(q, id=id))
def aliases(id):
entity = Entity.by_id(id)
pager = Pager(entity.aliases, id=id)
return jsonify(pager.to_dict())
def index():
collections = authz.collections(authz.READ)
enable_cache(vary_user=True, vary=collections)
q = Collection.all_by_ids(collections)
q = q.order_by(Collection.label.asc())
return jsonify(Pager(q))
def index():
collection_ids = match_ids('collection', authz.collections(authz.READ))
q = Entity.all()
q = q.filter(Entity.collection_id.in_(collection_ids))
return jsonify(Pager(q))
def index(collection_id):
collection = obj_or_404(Collection.by_id(collection_id))
authz.require(authz.collection_read(collection.id))
q = Network.all()
q = q.filter(Network.collection_id == collection.id)
return jsonify(Pager(q, collection_id=collection.id))
def index():
datasets = Dataset.all()
pager = Pager(datasets)
return jsonify(pager.to_dict())
def index():
authz.require(authz.is_admin())
crawlers = list(get_exposed_crawlers())
return jsonify(Pager(crawlers, limit=20))
def index():
datasets = Dataset.all()
pager = Pager(datasets)
return jsonify(pager.to_dict())
