def attributes():
etag_cache_keygen()
attributes = available_attributes(
request.args,
sources=authz.authz_sources('read'), # noqa
lists=authz.authz_lists('read')) # noqa
return jsonify(attributes)
def view(id):
authz.require(authz.list_read(id))
lst = obj_or_404(List.by_id(id))
etag_cache_keygen(lst)
data = lst.to_dict()
data['can_write'] = authz.list_write(id)
if data['can_write']:
data['users'] = [u.id for u in lst.users]
return jsonify(data)
def query():
etag_cache_keygen()
query = document_query(request.args, lists=authz.authz_lists('read'),
sources=authz.authz_sources('read'))
results = search_documents(query)
pager = Pager(results,
results_converter=lambda ds: [add_urls(d) for d in ds])
data = pager.to_dict()
data['facets'] = transform_facets(results.result.get('aggregations', {}))
return jsonify(data)
def index():
crawlers = []
for name, cls in get_crawlers().items():
crawlers.append({
'name': name,
'label': getattr(cls, 'LABEL') or name,
'url': getattr(cls, 'SITE'),
})
etag_cache_keygen([c['name'] for c in crawlers])
return jsonify({'results': crawlers, 'total': len(crawlers)})
def index():
sources = []
latest = set()
for source in Source.all_by_user(current_user):
data = source.to_dict()
data['can_write'] = authz.source_write(source.slug)
latest.add(data['updated_at'])
sources.append(data)
etag_cache_keygen(max(latest))
return jsonify({'results': sources, 'total': len(sources)})
def index():
crawlers = []
for name, cls in get_crawlers().items():
crawlers.append({
'name': name,
'label': getattr(cls, 'LABEL') or name,
'url': getattr(cls, 'SITE'),
})
etag_cache_keygen([c['name'] for c in crawlers])
return jsonify({'results': crawlers, 'total': len(crawlers)})
def view(slug):
authz.require(authz.source_read(slug))
source = obj_or_404(Source.by_slug(slug))
etag_cache_keygen(source)
data = source.to_dict()
data['can_write'] = authz.source_write(slug)
if data['can_write']:
data['users'] = [u.id for u in source.users]
data['config'] = source.config
return jsonify(data)
def view(slug):
authz.require(authz.source_read(slug) and authz.is_admin())
source = obj_or_404(Source.by_slug(slug))
etag_cache_keygen(source)
data = source.to_dict()
data['can_write'] = authz.source_write(slug)
if data['can_write']:
data['users'] = [u.id for u in source.users]
data['config'] = source.config
return jsonify(data)
def index():
sources = []
latest = set()
for source in Source.all_by_user(current_user):
data = source.to_dict()
data['can_write'] = authz.source_write(source.slug)
latest.add(data['updated_at'])
sources.append(data)
if len(latest):
etag_cache_keygen(max(latest))
return jsonify({'results': sources, 'total': len(sources)})
def query():
etag_cache_keygen()
graph = generate_graph(request.args)
format = request.args.get('format', '').lower().strip()
if format == 'gexf':
sio = StringIO()
nx.write_gexf(graph, sio)
sio.seek(0)
return send_file(sio, mimetype='application/xml')
else:
data = json_graph.node_link_data(graph)
data['partial'] = graph.partial
return jsonify(data)
def query():
etag_cache_keygen()
graph = generate_graph(request.args)
format = request.args.get('format', '').lower().strip()
if format == 'gexf':
sio = StringIO()
nx.write_gexf(graph, sio)
sio.seek(0)
return send_file(sio, mimetype='application/xml')
else:
data = json_graph.node_link_data(graph)
data['partial'] = graph.partial
return jsonify(data)
def records(document_id):
etag_cache_keygen()
document = get_document(document_id)
query = records_query(document.id, request.args)
if query is None:
return jsonify({
'status': 'ok',
'message': 'no query'
})
query['size'] = get_limit(default=30)
query['from'] = get_offset()
res = execute_records_query(document.id, request.args, query)
return jsonify(res)
def _query():
'''
everything here should be applicable both to the internal and to the
public api
'''
etag_cache_keygen()
query = document_query(request.args, lists=authz.authz_lists('read'),
sources=authz.authz_sources('read'),
highlights=True)
results = search_documents(query)
pager = Pager(results,
results_converter=lambda ds: [add_urls(d) for d in ds])
data = pager.to_dict()
#import ipdb; ipdb.set_trace()
data['facets'] = transform_facets(results.result.get('aggregations', {}))
return data
def metadata():
etag_cache_keygen()
country_names = {
'zz': 'Global',
'xk': 'Kosovo'
}
for country in countries:
country_names[country.alpha2.lower()] = country.name
language_names = dict(Locale('en').languages.items())
language_names = {k: v for k, v in language_names.items() if len(k) == 2}
return jsonify({
'status': 'ok',
'fields': CORE_FACETS,
'countries': country_names,
'languages': language_names
})
def _query():
'''
everything here should be applicable both to the internal and to the
public api
'''
etag_cache_keygen()
query = document_query(request.args,
lists=authz.authz_lists('read'),
sources=authz.authz_sources('read'),
highlights=True)
results = search_documents(query)
pager = Pager(results,
results_converter=lambda ds: [add_urls(d) for d in ds])
data = pager.to_dict()
#import ipdb; ipdb.set_trace()
data['facets'] = transform_facets(results.result.get('aggregations', {}))
return data
def view(id):
authz.require(authz.watchlist_read(id))
watchlist = obj_or_404(Watchlist.by_id(id))
etag_cache_keygen(watchlist)
return jsonify(watchlist)
def view(id):
authz.require(authz.source_read(id))
source = obj_or_404(Source.by_id(id))
etag_cache_keygen(source)
return jsonify(source)
def manifest(collection, package_id):
etag_cache_keygen(collection, package_id)
package = get_package(collection, package_id)
return jsonify(package.manifest)
def query():
etag_cache_keygen()
query = documents_query(request.args)
query['size'] = get_limit(default=100)
query['from'] = get_offset()
return jsonify(execute_documents_query(request.args, query))
def manifest(collection, package_id):
etag_cache_keygen(collection, package_id)
package = get_package(collection, package_id)
return jsonify(package.manifest)
def attributes():
etag_cache_keygen()
attributes = available_attributes(request.args,
sources=authz.authz_sources('read'), # noqa
lists=authz.authz_lists('read')) # noqa
return jsonify(attributes)
