def trigger_reset():
"""
Allow user to trigger a reset of the password in case they forget it
"""
email = request_data().get('email')
# Simple check to see if the email was provided. Flash error if not
if email is None or not len(email):
return jsonify(
{
'status': 'error',
'message': _("Please enter an email address!")
},
status=400)
account = Account.by_email(email)
# If no account is found we let the user know that it's not registered
if account is None:
return jsonify(
{
'status': 'error',
'message': _("No user is registered under this address!")
},
status=400)
# Send the reset link to the email of this account
send_reset_link(account)
return jsonify({
'status':
'ok',
'message':
_("You've received an email with a link to reset your "
"password. Please check your inbox.")
})
def load_next():
authz.require(authz.system_edit())
next = request_pairing(exclude=request.args.getlist("exclude"))
db.session.commit()
if next is None:
return jsonify({"status": "done"})
return jsonify({"status": "next", "next": next.id})
def reset_password():
user = User.by_email(request_data().get('email'))
if user is None:
message = {'email': 'This email address is not linked to a user.'}
return jsonify({'status': 400, 'errors': message}, status=400)
send_reset_link(user)
return jsonify({'status': 200})
def reconcile(dataset):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
dataset = Dataset.by_name(dataset)
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(dataset, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(dataset, q)
return jsonify(queries)
else:
return reconcile_index(dataset)
def reconcile(dataset):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
dataset = Dataset.by_name(dataset)
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(dataset, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(dataset, q)
return jsonify(queries)
else:
return reconcile_index(dataset)
def initiate():
authz.require(authz.system_edit())
root = request_data().get("root")
if root is None:
return jsonify({"status": "error", "message": "No root entity provided."}, status=400)
enrich_entity.delay(root, root, request_data().get("spider"))
return jsonify({"status": "ok", "root": root})
def ooemail_authorized():
'''
This is a callback for when we are returning from the
external auth provider.
So, we use this to handle our sign-in
'''
usr = Role.by_email(request.args.get('email'))
if usr is None:
return jsonify({
'status': 'error',
'message': 'You are not authorized to do this.',
'roles': request.auth_roles,
'user': request.auth_role
}, status=403)
ok = usr.check_pw(request.args.get('password'))
if ok:
store_login(usr)
return 'oo email authorized'
else:
return jsonify({
'status': 'error',
'message': 'You are not authorized to do this.',
'roles': request.auth_roles,
'user': request.auth_role
}, status=403)
def trigger_reset():
"""
Allow user to trigger a reset of the password in case they forget it
"""
email = request_data().get('email')
# Simple check to see if the email was provided. Flash error if not
if email is None or not len(email):
return jsonify({
'status': 'error',
'message': _("Please enter an email address!")
}, status=400)
account = Account.by_email(email)
# If no account is found we let the user know that it's not registered
if account is None:
return jsonify({
'status': 'error',
'message': _("No user is registered under this address!")
}, status=400)
# Send the reset link to the email of this account
send_reset_link(account)
return jsonify({
'status': 'ok',
'message': _("You've received an email with a link to reset your "
"password. Please check your inbox.")
})
def reconcile():
"""
Reconciliation API, emulates Google Refine API.
See: http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
# authz.require(authz.system_read())
data = request.args.copy()
data.update(request.form.copy())
log_event(request)
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(q)
return jsonify(queries)
else:
return reconcile_index()
def suggest_nodes():
prefix = request.args.get('prefix', '').strip()
collection_id = [int(c) for c in request.args.getlist('collection_id')]
if len(prefix) < 3 and not len(collection_id):
return jsonify({'status': 'ok', 'nodes': []})
resp = queries.suggest_nodes(get_graph(), collection_id, prefix,
get_limit(default=20), get_offset())
return jsonify(resp)
def index():
collections = authz.collections(authz.READ)
if not len(collections):
return jsonify(Pager([]))
q = session.query(Source)
q = q.filter(Source.id.in_(collections))
q = q.order_by(Source.title.asc())
return jsonify(Pager(q))
def similar(id):
entity, _ = get_entity(id, request.authz.READ)
schema = schemata.get(entity.get('schema'))
if not schema.fuzzy:
return jsonify({'status': 'ignore', 'results': [], 'total': 0})
state = QueryState(request.args, request.authz)
combined = combined_entity(entity)
return jsonify(similar_entities(combined, state))
def login():
data = request_data()
account = Account.by_name(data.get("login"))
if account is not None:
if check_password_hash(account.password, data.get("password")):
login_user(account, remember=True)
return jsonify({"status": "ok", "message": _("Welcome back, %(name)s!", name=account.name)})
return jsonify({"status": "error", "errors": {"password": _("Incorrect user name or password!")}}, status=400)
def login():
data = request_data()
user = User.by_email(data.get('email'))
if user is not None and user.verify(data.get('password')):
login_user(user, remember=True)
return jsonify({'status': 200, 'user': user})
message = {'password': '******'}
return jsonify({'status': 400, 'errors': message}, status=400)
def suggest_nodes():
prefix = request.args.get('prefix', '').strip()
collection_id = [int(c) for c in request.args.getlist('collection_id')]
if len(prefix) < 3 and not len(collection_id):
return jsonify({'status': 'ok', 'nodes': []})
resp = queries.suggest_nodes(get_graph(), collection_id, prefix,
get_limit(default=20), get_offset())
return jsonify(resp)
def peek():
if not get_config('ALLOW_PEEKING', True):
return jsonify({'active': False})
enable_cache(vary_user=True)
state = QueryState(request.args, request.authz)
response = peek_query(state)
if not request.authz.logged_in:
response.pop('roles', None)
return jsonify(response)
def index():
collections = authz.collections(authz.READ)
if not len(collections):
return jsonify(Pager([]))
q = session.query(Collection)
q = q.filter(Collection.id.in_(collections))
q = q.options(subqueryload('subjects'))
q = q.order_by(Collection.title.asc())
return jsonify(Pager(q))
def review(dataset):
entities = Entity.all()
dataset = Dataset.find(dataset)
entities = entities.filter_by(dataset=dataset)
entities = entities.filter(Entity.reviewed == False) # noqa
review_count = entities.count()
if review_count == 0:
return jsonify(None)
entities = entities.offset(randint(0, review_count - 1))
return jsonify(entities.first())
def complete(format='json'):
if not current_user.is_authenticated():
msg = _("You are not authorized to see that page")
return jsonify({'status': 'error', 'message': msg}, status=403)
query = db.session.query(Account)
filter_string = request.args.get('q', '') + '%'
query = query.filter(or_(Account.name.ilike(filter_string),
Account.fullname.ilike(filter_string)))
return jsonify(Pager(query))
def review(dataset):
entities = Entity.all()
dataset = Dataset.find(dataset)
entities = entities.filter_by(dataset=dataset)
entities = entities.filter(Entity.reviewed == False) # noqa
review_count = entities.count()
if review_count == 0:
return jsonify(None)
entities = entities.offset(randint(0, review_count - 1))
return jsonify(entities.first())
def load_next(root):
authz.require(authz.system_edit())
q = Context.by_root(root)
q = q.filter(Context.enrich_status == PENDING)
q = q.order_by(Context.enrich_score.desc())
context = q.first()
if context is None:
# TODO: spider status system
return jsonify({"status": "wait"})
return jsonify({"status": "next", "next": context.id})
def complete(format='json'):
if not current_user.is_authenticated():
msg = _("You are not authorized to see that page")
return jsonify({'status': 'error', 'message': msg}, status=403)
query = db.session.query(Account)
filter_string = request.args.get('q', '') + '%'
query = query.filter(or_(Account.name.ilike(filter_string),
Account.fullname.ilike(filter_string)))
return jsonify(Pager(query))
def query():
authz.require(authz.system_read())
if request.method == "GET":
try:
q = json.loads(request.args.get("q"))
except (TypeError, ValueError) as e:
data = {"status": "error", "message": "Invalid query: %s" % unicode(e)}
return jsonify(data, status=400)
else:
q = request_data()
return jsonify(execute_query(q))
def queue():
authz.require(authz.is_admin())
data = request_data()
crawler_id = data.get('crawler_id')
for cls in get_exposed_crawlers():
if crawler_id == cls.get_id():
incremental = bool(data.get('incremental', False))
execute_crawler.delay(crawler_id, incremental=incremental)
return jsonify({'status': 'queued'})
return jsonify({'status': 'error', 'message': 'No such crawler'},
status=400)
def authz():
obj = Dataset.by_name(request.args.get('dataset'))
etag_cache_keygen(obj, private=True)
if obj is None:
return jsonify({
'read': False,
'update': False
})
return jsonify({
'read': dataset.read(obj),
'update': dataset.update(obj)
})
def dc_projects():
slug = request.args.get('source')
authz.require(authz.source_read(slug))
source = obj_or_404(Source.by_slug(slug))
if not isinstance(source.crawler_instance, DocumentCloudCrawler):
return jsonify({'credentials': False})
username = request.args.get('username')
password = request.args.get('password')
projects = source.crawler_instance.get_projects(username, password)
if projects is False:
return jsonify({'credentials': False})
else:
return jsonify({'credentials': True, 'projects': projects})
def records(document_id):
document = get_document(document_id)
enable_cache(vary_user=True)
query = records_query(document.id, request.args)
if query is None:
return jsonify({"status": "ok", "message": "no query"})
query["size"] = get_limit(default=30)
query["from"] = get_offset()
result = execute_records_query(query)
params = next_params(request.args, result)
if params is not None:
result["next"] = url_for("search_api.record", document_id=document_id, **params)
return jsonify(result)
def records(document_id):
etag_cache_keygen()
document = get_document(document_id)
query = records_query(document.id, request.args)
if query is None:
return jsonify({
'status': 'ok',
'message': 'no query'
})
query['size'] = get_limit(default=30)
query['from'] = get_offset()
res = execute_records_query(document.id, request.args, query)
return jsonify(res)
def dc_projects():
slug = request.args.get('source')
authz.require(authz.source_read(slug))
source = obj_or_404(Source.by_slug(slug))
if not isinstance(source.crawler_instance, DocumentCloudCrawler):
return jsonify({'credentials': False})
username = request.args.get('username')
password = request.args.get('password')
projects = source.crawler_instance.get_projects(username, password)
if projects is False:
return jsonify({'credentials': False})
else:
return jsonify({'credentials': True,
'projects': projects})
def suggest():
request.authz.require(request.authz.logged_in)
prefix = stringify(request.args.get('prefix'))
if prefix is None or len(prefix) < 3:
# Do not return 400 because it's a routine event.
return jsonify({
'status': 'error',
'message': 'prefix filter is too short',
'results': [],
'total': 0
})
# this only returns users, not groups
q = Role.by_prefix(prefix)
return jsonify(Pager(q, limit=10))
def store(root):
authz.require(authz.system_edit())
ctx_data = request_data()
q = Context.by_root(root)
q = q.filter(Context.id == ctx_data.get("id"))
context = q.first()
if context is None:
return jsonify({"status": "failed"}, status=400)
context.update(request_data())
context.active = context.enrich_status == ACCEPTED
context.user = current_user
db.session.commit()
# TODO: task newly activated entities.
return jsonify({"status": "ok"})
def pending():
q = db.session.query(Entity)
q = q.filter(Entity.state == Entity.STATE_PENDING)
clause = Collection.id.in_(authz.collections(authz.READ))
q = q.filter(Entity.collections.any(clause))
ref = aliased(Reference)
q = q.join(ref)
q = q.group_by(Entity)
q = q.order_by(func.sum(ref.weight).desc())
entity = q.first()
if entity is None:
return jsonify({'empty': True})
data = entity.to_dict()
data['name_latin'] = latinize_text(data['name'], lowercase=False)
return jsonify(data)
def records(document_id):
document = get_document(document_id)
enable_cache(vary_user=True)
query = records_query(document.id, request.args)
if query is None:
return jsonify({'status': 'ok', 'message': 'no query'})
query['size'] = get_limit(default=30)
query['from'] = get_offset()
result = execute_records_query(query)
params = next_params(request.args, result)
if params is not None:
result['next'] = url_for('search_api.record',
document_id=document_id,
**params)
return jsonify(result)
def pending():
q = db.session.query(Entity)
q = q.filter(Entity.state == Entity.STATE_PENDING)
clause = Collection.id.in_(authz.collections(authz.READ))
q = q.filter(Entity.collections.any(clause))
ref = aliased(Reference)
q = q.join(ref)
q = q.group_by(Entity)
q = q.order_by(func.sum(ref.weight).desc())
entity = q.first()
if entity is None:
return jsonify({'empty': True})
data = entity.to_dict()
data['name_latin'] = latinize_text(data['name'], lowercase=False)
return jsonify(data)
def structure(name):
dataset = get_dataset(name)
etag_cache_keygen(dataset)
return jsonify({
'fields': dataset.fields,
'samples': dataset.samples
})
def delete(id):
request.authz.require(request.authz.session_write())
alert = obj_or_404(Alert.by_id(id, role=request.authz.role))
alert.delete()
db.session.commit()
log_event(request)
return jsonify({'status': 'ok'})
def metadata():
enable_cache(server_side=False)
schemata = {}
for schema_id, schema in resolver.store.items():
if not schema_id.endswith('#'):
schema_id = schema_id + '#'
schemata[schema_id] = {
'id': schema_id,
'title': schema.get('title'),
'faIcon': schema.get('faIcon'),
'plural': schema.get('plural', schema.get('title')),
'description': schema.get('description'),
'inline': schema.get('inline', False)
}
return jsonify({
'status': 'ok',
'app': {
'title': get_app_title(),
'url': get_app_url(),
'samples': get_config('SAMPLE_SEARCHES')
},
'fields': Metadata.facets(),
'categories': Collection.CATEGORIES,
'countries': COUNTRY_NAMES,
'languages': LANGUAGE_NAMES,
'schemata': schemata
})
def ingest_upload(collection_id):
collection = obj_or_404(Collection.by_id(collection_id))
request.authz.require(request.authz.collection_write(collection.id))
log_event(request)
try:
meta = json.loads(request.form.get('meta', '{}'))
meta['crawler_id'] = 'user_upload:%s' % request.authz.role.id
meta['crawler_run'] = make_textid()
except Exception as ex:
raise BadRequest(unicode(ex))
metas = []
for storage in request.files.values():
file_meta = meta.copy()
file_meta['mime_type'] = storage.mimetype
file_meta['file_name'] = storage.filename
file_meta['source_path'] = storage.filename
validate(file_meta, 'metadata.json#')
file_meta = Metadata.from_data(file_meta)
sec_fn = os.path.join(upload_folder, secure_filename(storage.filename))
storage.save(sec_fn)
ingest_file(collection_id, file_meta, sec_fn, move=True,
queue=USER_QUEUE, routing_key=USER_ROUTING_KEY)
metas.append(file_meta)
return jsonify({'status': 'ok', 'metadata': metas})
def set_locale():
locale = request.json.get('locale')
if locale is not None:
session['locale'] = locale
session.modified = True
return jsonify({'locale': locale})
def view(id):
authz.require(authz.logged_in())
role = obj_or_404(Role.by_id(id))
data = role.to_dict()
if role.id != request.auth_role.id:
del data["email"]
return jsonify(data)
def register():
""" Perform registration of a new user """
disable_cache()
require.account.create()
data = AccountRegister().deserialize(request_data())
# Check if the username already exists, return an error if so
if Account.by_name(data['name']):
raise colander.Invalid(
AccountRegister.name,
_("Login name already exists, please choose a "
"different one"))
# Check if passwords match, return error if not
if not data['password1'] == data['password2']:
raise colander.Invalid(AccountRegister.password1,
_("Passwords don't match!"))
# Create the account
account = Account()
account.name = data['name']
account.fullname = data['fullname']
account.email = data['email']
account.public_email = data['public_email']
account.password = generate_password_hash(data['password1'])
db.session.add(account)
db.session.commit()
# Perform a login for the user
login_user(account, remember=True)
# Registration successful - Redirect to the front page
return jsonify(account)
def suggest(dataset):
"""
Suggest API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/SuggestApi
"""
dataset = Dataset.by_name(dataset)
entities = Entity.all().filter(Entity.invalid != True) # noqa
query = request.args.get('prefix', '').strip()
entities = entities.filter(Entity.name.ilike('%s%%' % query))
entities = entities.offset(get_offset(field='start'))
entities = entities.limit(get_limit(default=20))
matches = []
for entity in entities:
matches.append({
'name': entity.name,
'n:type': {
'id': '/' + dataset.name,
'name': dataset.label
},
'id': entity.id
})
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": query,
"result": matches
})
def create_or_update(collection=None, source=None):
if collection is not None:
authz.require(authz.collection(authz.WRITE, collection))
if source is not None:
authz.require(authz.source(authz.WRITE, source))
resource_type = Permission.COLLECTION if collection else Permission.SOURCE
resource_id = collection or source
data = request_data()
validate(data, permissions_schema)
# check that the role exists.
rq = session.query(Role).filter(Role.id == data['role'])
if rq.first() is None:
raise BadRequest()
q = session.query(Permission)
q = q.filter(Permission.role_id == data['role'])
q = q.filter(Permission.resource_type == resource_type)
q = q.filter(Permission.resource_id == resource_id)
permission = q.first()
if permission is None:
permission = Permission()
permission.role_id = data['role']
permission.resource_type = resource_type
permission.resource_id = resource_id
permission.read = data['read']
permission.write = data['write']
session.add(permission)
session.commit()
return jsonify({
'status': 'ok',
'updated': permission
})
def delete(id):
authz.require(authz.logged_in())
alert = obj_or_404(Alert.by_id(id))
authz.require(alert.user_id == current_user.id)
db.session.delete(alert)
db.session.commit()
return jsonify({'status': 'ok'})
def delete(id):
entity = obj_or_404(Entity.by_id(id))
authz.require(authz.collection_write(entity.collection_id))
entity.delete()
db.session.commit()
analyze_entity.delay(id)
return jsonify({'status': 'ok'})
def ping():
from spendb.tasks import ping
ping.delay()
return jsonify({
'status': 'ok',
'message': gettext("Sent ping!")
})
def delete(name):
dataset = get_dataset(name)
require.dataset.update(dataset)
dataset.fact_table.drop()
db.session.delete(dataset)
db.session.commit()
return jsonify({'status': 'deleted'}, status=410)
def permissions_index(collection):
request.authz.require(request.authz.collection_write(collection))
q = Permission.all()
q = q.filter(Permission.collection_id == collection)
permissions = []
roles_seen = set()
for permission in q.all():
if check_visible(permission.role):
permissions.append(permission)
roles_seen.add(permission.role.id)
# this workaround ensures that all groups are visible for the user to
# select in the UI even if they are not currently associated with the
# collection.
for role in Role.all_groups():
if check_visible(role):
if role.id not in roles_seen:
roles_seen.add(role.id)
permissions.append({
'write': False,
'read': False,
'role': role,
'role_id': role.id
})
return jsonify({'total': len(permissions), 'results': permissions})
def view(id):
authz.require(authz.logged_in())
role = obj_or_404(Role.by_id(id))
data = role.to_dict()
if role.id != request.auth_role.id:
del data['email']
return jsonify(data)
def set_locale():
disable_cache()
locale = request.json.get('locale')
if locale is not None:
set_session_locale(locale)
return jsonify({'locale': locale})
def table(document_id, table_id):
document = get_document(document_id)
enable_cache(vary_user=True)
try:
return jsonify(document.meta.tables[table_id])
except IndexError:
raise NotFound("No such table: %s" % table_id)
def delete(id):
authz.require(authz.watchlist_write(id))
watchlist = obj_or_404(Watchlist.by_id(id))
analyze_terms.delay(watchlist.terms)
watchlist.delete()
db.session.commit()
return jsonify({'status': 'ok'})
def index():
q = entities_query(request.args)
q['size'] = get_limit(default=50)
q['from'] = get_offset()
doc_counts = arg_bool('doc_counts')
res = execute_entities_query(request.args, q, doc_counts=doc_counts)
return jsonify(res)
def update(account):
""" Change settings for the logged in user """
require.account.update(current_user)
data = AccountSettings().deserialize(request_data())
# If the passwords don't match we notify the user
if not data['password1'] == data['password2']:
raise colander.Invalid(AccountSettings.password1,
_("Passwords don't match!"))
current_user.fullname = data['fullname']
current_user.email = data['email']
current_user.public_email = data['public_email']
if data['twitter'] is not None:
current_user.twitter_handle = data['twitter'].lstrip('@')
current_user.public_twitter = data['public_twitter']
# If a new password was provided we update it as well
if data['password1'] is not None and len(data['password1']):
current_user.password = generate_password_hash(data['password1'])
# Do the actual update in the database
db.session.add(current_user)
db.session.commit()
return jsonify(current_user)
def all():
q = Entity.all()
q = q.filter(Entity.state == Entity.STATE_ACTIVE)
clause = Collection.id.in_(authz.collections(authz.READ))
q = q.filter(Entity.collections.any(clause))
q = q.order_by(Entity.id.asc())
return jsonify(Pager(q, limit=100))
