def index(): collections = authz.collections(authz.READ) if not len(collections): return jsonify(Pager([])) q = session.query(Source) q = q.filter(Source.id.in_(collections)) q = q.order_by(Source.title.asc()) return jsonify(Pager(q))
def index(): collections = authz.collections(authz.READ) if not len(collections): return jsonify(Pager([])) q = session.query(Collection) q = q.filter(Collection.id.in_(collections)) q = q.options(subqueryload('subjects')) q = q.order_by(Collection.title.asc()) return jsonify(Pager(q))
def match(): dataset_arg = request.args.get('dataset') dataset = Dataset.find(dataset_arg) matches = find_matches(dataset, request.args.get('name'), filter=request.args.get('filter'), exclude=arg_int('exclude')) pager = Pager(matches) return jsonify(pager.to_dict())
def index(): q = List.all_by_user(current_user) data = Pager(q).to_dict() results = [] for lst in data.pop('results'): ldata = lst.to_dict() ldata['can_write'] = authz.list_write(lst.id) results.append(ldata) data['results'] = results return jsonify(data)
def index(): q = Collection.all_by_user(current_user) data = Pager(q).to_dict() results = [] for lst in data.pop('results'): ldata = lst.to_dict() ldata['permissions'] = {'write': authz.collection_write(lst.id)} results.append(ldata) data['results'] = results return jsonify(data)
def query(): etag_cache_keygen() query = document_query(request.args, lists=authz.authz_lists('read'), sources=authz.authz_sources('read')) results = search_documents(query) pager = Pager(results, results_converter=lambda ds: [add_urls(d) for d in ds]) data = pager.to_dict() data['facets'] = transform_facets(results.result.get('aggregations', {})) return jsonify(data)
def index(): q = Collection.all_by_user(current_user) data = Pager(q).to_dict() results = [] for lst in data.pop('results'): ldata = lst.to_dict() ldata['permissions'] = { 'write': authz.collection_write(lst.id) } results.append(ldata) data['results'] = results return jsonify(data)
def load_relations(entity, id, slug): relation_sections = [] q = grano.relations.query().limit(0) q = q.filter('facet', 'schema').filter('entity', id) schema_types = q.data.get('facets', {}).get('schema', {}) for (schema, count) in schema_types.get('results', []): iq = grano.relations.query().limit(15) iq = iq.filter('entity', id) iq = iq.filter('schema', schema.get('name')) pager = Pager(iq, name=schema.get('name'), id=id, slug=slug) relations = [] for r in pager: r.props = Properties(r) r.other = r.source if r.target.id == id else r.target relations.append(r) data = { 'schema': schemata.by_name(schema.get('name')), 'count': count, 'pager': pager, 'relations': relations } relation_sections.append(data) return sorted(relation_sections, key=lambda r: r['schema'].label)
def index(dataset): dataset = get_dataset(dataset) q = Run.all(dataset) if 'source' in request.args: q = q.filter(Run.source == request.args.get('source')) pager = Pager(q, dataset=dataset.name) return jsonify(pager)
def index(): sources_ids = match_ids('sources', authz.sources(authz.READ)) q = Document.all().filter(Document.source_id.in_(sources_ids)) hashes = request.args.getlist('content_hash') if len(hashes): q = q.filter(Document.content_hash.in_(hashes)) return jsonify(Pager(q))
def query_index(): q = Dataset.all_by_account(current_user, order=False) q = q.order_by(Dataset.updated_at.desc()) # Filter by languages if they have been provided for language in request.args.getlist('languages'): l = aliased(DatasetLanguage) q = q.join(l, Dataset._languages) q = q.filter(l.code == language) # Filter by territories if they have been provided for territory in request.args.getlist('territories'): t = aliased(DatasetTerritory) q = q.join(t, Dataset._territories) q = q.filter(t.code == territory) # Filter by account if one has been provided for account in request.args.getlist('account'): a = aliased(Account) q = q.join(a, Dataset.managers) q = q.filter(a.name == account) # Return a list of languages as dicts with code, count, url and label languages = [{'code': code, 'count': count, 'label': LANGUAGES.get(code)} for (code, count) in DatasetLanguage.dataset_counts(q)] territories = [{'code': code, 'count': count, 'label': COUNTRIES.get(code)} for (code, count) in DatasetTerritory.dataset_counts(q)] pager = Pager(q, limit=15) return pager, languages, territories
def all(): q = Entity.all() q = q.filter(Entity.state == Entity.STATE_ACTIVE) clause = Collection.id.in_(authz.collections(authz.READ)) q = q.filter(Entity.collections.any(clause)) q = q.order_by(Entity.id.asc()) return jsonify(Pager(q, limit=100))
def _query(): ''' everything here should be applicable both to the internal and to the public api ''' etag_cache_keygen() query = document_query(request.args, lists=authz.authz_lists('read'), sources=authz.authz_sources('read'), highlights=True) results = search_documents(query) pager = Pager(results, results_converter=lambda ds: [add_urls(d) for d in ds]) data = pager.to_dict() #import ipdb; ipdb.set_trace() data['facets'] = transform_facets(results.result.get('aggregations', {})) return data
def index(): collection_ids = match_ids('collection', authz.collections(authz.READ)) q = Document.all() clause = Collection.id.in_(collection_ids) q = q.filter(Document.collections.any(clause)) hashes = request.args.getlist('content_hash') if len(hashes): q = q.filter(Document.content_hash.in_(hashes)) return jsonify(Pager(q))
def index(): authz = request.authz collections = request.args.getlist('collection') collections = authz.collections_intersect(authz.READ, collections) q = Document.all() q = q.filter(Document.collection_id.in_(collections)) hashes = request.args.getlist('content_hash') if len(hashes): q = q.filter(Document.content_hash.in_(hashes)) return jsonify(Pager(q))
def index(dataset): dataset = get_dataset(dataset) package = data_manager.package(dataset.name) sources = list(package.all(Source)) sources = sorted(sources, key=lambda s: s.meta.get('updated_at'), reverse=True) rc = lambda ss: [source_to_dict(dataset, s) for s in ss] return jsonify( Pager(sources, dataset=dataset.name, limit=5, results_converter=rc))
def references(document_id): doc = get_document(document_id) q = db.session.query(Reference) q = q.filter(Reference.document_id == doc.id) q = q.filter(Reference.origin == 'regex') q = q.join(Entity) q = q.filter(Entity.state == Entity.STATE_ACTIVE) q = q.filter(Entity.collection_id.in_(request.authz.collections_read)) q = q.order_by(Reference.weight.desc()) return jsonify(Pager(q, document_id=document_id))
def complete(format='json'): if not current_user.is_authenticated(): msg = _("You are not authorized to see that page") return jsonify({'status': 'error', 'message': msg}, status=403) query = db.session.query(Account) filter_string = request.args.get('q', '') + '%' query = query.filter(or_(Account.name.ilike(filter_string), Account.fullname.ilike(filter_string))) return jsonify(Pager(query))
def states(): authz.require(authz.is_admin()) q = db.session.query(CrawlerState) q = q.filter(CrawlerState.status == CrawlerState.STATUS_FAIL) if 'crawler_id' in request.args: q = q.filter(CrawlerState.crawler_id == request.args.get('crawler_id')) if 'crawler_run' in request.args: q = q.filter(CrawlerState.crawler_run == request.args.get('crawler_run')) if 'error_type' in request.args: q = q.filter(CrawlerState.error_type == request.args.get('error_type')) q = q.order_by(CrawlerState.created_at.desc()) response = Pager(q).to_dict() return jsonify(response)
def index(): list_ids = List.user_list_ids(current_user) filter_lists = request.args.getlist('list') if len(filter_lists): try: filter_lists = [int(f) for f in filter_lists] list_ids = [l for l in list_ids if l in filter_lists] except ValueError: raise BadRequest() prefix = request.args.get('prefix') q = Entity.by_lists(list_ids, prefix=prefix) return jsonify(Pager(q))
def index(): entities = Entity.all() dataset_arg = request.args.get("dataset") if dataset_arg is not None: dataset = Dataset.find(dataset_arg) entities = entities.filter_by(dataset=dataset) filter_name = request.args.get("filter_name", "") if len(filter_name): query = "%" + filter_name + "%" entities = entities.filter(Entity.name.ilike(query)) # TODO, other filters. format = request.args.get("format", "json").lower().strip() if format == "csv": res = csvify(entities) else: pager = Pager(entities) res = jsonify(pager.to_dict()) if arg_bool("download"): fn = dataset_filename(dataset, format) res.headers["Content-Disposition"] = "attachment; filename=" + fn return res
def index(): entities = Entity.all() dataset_arg = request.args.get('dataset') if dataset_arg is not None: dataset = Dataset.find(dataset_arg) entities = entities.filter_by(dataset=dataset) filter_name = request.args.get('filter_name', '') if len(filter_name): query = '%' + filter_name + '%' entities = entities.filter(Entity.name.ilike(query)) # TODO, other filters. format = request.args.get('format', 'json').lower().strip() if format == 'csv': res = csvify(entities) else: pager = Pager(entities) res = jsonify(pager.to_dict()) if arg_bool('download'): fn = dataset_filename(dataset, format) res.headers['Content-Disposition'] = 'attachment; filename=' + fn return res
def index(): collection_ids = Collection.user_ids(current_user) filter_collections = request.args.getlist('collection') if len(filter_collections): try: collection_ids = [ l for l in collection_ids if l in filter_collections ] except ValueError: raise BadRequest() prefix = request.args.get('prefix') q = Entity.by_collection(collection_ids, prefix=prefix) return jsonify(Pager(q))
def suggest(): request.authz.require(request.authz.logged_in) prefix = stringify(request.args.get('prefix')) if prefix is None or len(prefix) < 3: # Do not return 400 because it's a routine event. return jsonify({ 'status': 'error', 'message': 'prefix filter is too short', 'results': [], 'total': 0 }) # this only returns users, not groups q = Role.by_prefix(prefix) return jsonify(Pager(q, limit=10))
def index(): try: authorized = authz.collections(authz.READ) collection_ids = [int(f) for f in request.args.getlist('collection')] collection_ids = collection_ids or authorized collection_ids = [c for c in collection_ids if c in authorized] except ValueError: raise BadRequest() q = Document.all() clause = Collection.id.in_(collection_ids) q = q.filter(Document.collections.any(clause)) hashes = request.args.getlist('content_hash') if len(hashes): q = q.filter(Document.content_hash.in_(hashes)) return jsonify(Pager(q))
def paths(id): collection = obj_or_404(Collection.by_id(id)) authz.require(authz.collection_read(collection.id)) start_entity_id = request.args.get('entity_id') labels = request.args.getlist('label') types = request.args.getlist('type') collection_id = request.args.getlist('collection_id') end_collection_id = authz.collections_intersect(authz.READ, collection_id) q = Path.find(collection, start_entity_id=start_entity_id, labels=labels, types=types, end_collection_id=end_collection_id) data = Pager(q, id=collection.id).to_dict() data['facets'] = Path.facets(collection, start_entity_id=start_entity_id, labels=labels, types=types, end_collection_id=end_collection_id, collection_id=authz.collections(authz.READ)) return jsonify(data)
def view(): query = grano.entities.query() query = query.filter('q', request.args.get('q', '')) schema_active = request.args.getlist('schema') main_schema = None if not len(schema_active): # TODO: get this from schema 'meta': schema_active = [ 'Person', 'Organization', 'Company', 'NonProfit', 'EducationalInstitution' ] else: main_schema = schemata.by_name(schema_active[0]) query = query.filter('schema', schema_active) query = query.filter('sort', '-degree') pager = Pager(query) return render_template('browser.html', main_schema=main_schema, pager=pager)
def index(): # allow to filter for writeable collections only, needed # in some UI scenarios: state = QueryState(request.args, request.authz) permission = request.args.get('permission') if permission not in [request.authz.READ, request.authz.WRITE]: permission = request.authz.READ collections = request.authz.collections[permission] # Other filters for navigation label = request.args.get('label') managed = state.getbool('managed', None) # Include counts (of entities, documents) in list view? counts = state.getbool('counts', False) def converter(colls): return [c.to_dict(counts=counts) for c in colls] facet = [f.lower().strip() for f in request.args.getlist('facet')] q = Collection.find(label=label, countries=state.getfilter('countries'), category=state.getfilter('category'), collection_id=collections, managed=managed) data = Pager(q).to_dict(results_converter=converter) facets = {} if 'countries' in facet: facets['countries'] = { 'values': Collection.facet_by(q, Collection.countries, mapping=COUNTRY_NAMES) } if 'category' in facet: mapping = get_config('COLLECTION_CATEGORIES', {}) facets['category'] = { 'values': Collection.facet_by(q, Collection.category, mapping=mapping) } data['facets'] = facets return jsonify(data)
def collection_crawlerstates(id): request.authz.require(request.authz.collection_read(id)) q = db.session.query(CrawlerState) q = q.filter(CrawlerState.collection_id == id) q = q.filter(or_( CrawlerState.error_type != 'init', CrawlerState.error_type == None # noqa )) status = request.args.get('status') if status: q = q.filter(CrawlerState.status == status) crawler_id = request.args.get('crawler_id') if crawler_id: q = q.filter(CrawlerState.crawler_id == crawler_id) crawler_run = request.args.get('crawler_run') if crawler_run: q = q.filter(CrawlerState.crawler_run == crawler_run) q = q.order_by(CrawlerState.created_at.desc()) return jsonify(Pager(q, id=id))
def aliases(id): entity = Entity.by_id(id) pager = Pager(entity.aliases, id=id) return jsonify(pager.to_dict())
def index(): collections = authz.collections(authz.READ) enable_cache(vary_user=True, vary=collections) q = Collection.all_by_ids(collections) q = q.order_by(Collection.label.asc()) return jsonify(Pager(q))
def index(): collection_ids = match_ids('collection', authz.collections(authz.READ)) q = Entity.all() q = q.filter(Entity.collection_id.in_(collection_ids)) return jsonify(Pager(q))
def index(collection_id): collection = obj_or_404(Collection.by_id(collection_id)) authz.require(authz.collection_read(collection.id)) q = Network.all() q = q.filter(Network.collection_id == collection.id) return jsonify(Pager(q, collection_id=collection.id))
def index(): datasets = Dataset.all() pager = Pager(datasets) return jsonify(pager.to_dict())
def index(): authz.require(authz.is_admin()) crawlers = list(get_exposed_crawlers()) return jsonify(Pager(crawlers, limit=20))