def reconcile(dataset):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
dataset = Dataset.by_name(dataset)
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(dataset, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(dataset, q)
return jsonify(queries)
else:
return reconcile_index(dataset)
def review(dataset):
entities = Entity.all()
dataset = Dataset.find(dataset)
entities = entities.filter_by(dataset=dataset)
entities = entities.filter(Entity.reviewed == False)
review_count = entities.count()
if review_count == 0:
return jsonify(None)
entities = entities.offset(randint(0, review_count - 1))
return jsonify(entities.first())
def review(dataset):
entities = Entity.all()
dataset = Dataset.find(dataset)
entities = entities.filter_by(dataset=dataset)
entities = entities.filter(Entity.reviewed==False)
review_count = entities.count()
if review_count == 0:
return jsonify(None)
entities = entities.offset(randint(0, review_count-1))
return jsonify(entities.first())
def status():
return jsonify({
'logged_in': authz.logged_in(),
'api_key': request.account.api_key if authz.logged_in() else None,
'account': request.account,
'base_url': url_for('index', _external=True)
})
def suggest(dataset):
"""
Suggest API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/SuggestApi
"""
dataset = Dataset.by_name(dataset)
entities = Entity.all().filter(Entity.invalid!=True)
query = request.args.get('prefix', '').strip()
entities = entities.filter(Entity.name.ilike('%s%%' % query))
entities = entities.offset(get_offset(field='start'))
entities = entities.limit(get_limit(default=20))
matches = []
for entity in entities:
matches.append({
'name': entity.name,
'n:type': {
'id': '/' + dataset.name,
'name': dataset.label
},
'id': entity.id
})
return jsonify({
"code" : "/api/status/ok",
"status" : "200 OK",
"prefix" : query,
"result" : matches
})
def query_pager(q, paginate=True, serializer=lambda x: x, **kw):
limit = get_limit()
offset = get_offset()
if paginate:
results = q.offset(offset).limit(limit)
else:
results = q
url = url_for(request.endpoint, _external=True, **kw)
count = q.count()
data = {
'count': count,
'limit': limit,
'offset': offset,
'format': url + args('LIMIT', 'OFFSET'),
'previous': prev_url(url, count, offset, limit),
'next': next_url(url, count, offset, limit),
'results': map(serializer, results)
}
response = jsonify(data, refs=True)
if data['next']:
response.headers.add_header('Link', '<%s>; rel=next' % data['next'])
if data['previous']:
response.headers.add_header('Link',
'<%s>; rel=previous' % data['previous'])
return response
def reconcile_index(dataset):
domain = url_for('index', _external=True).strip('/')
urlp = domain + '/entities/{{id}}'
meta = {
'name': 'nomenklatura: %s' % dataset.label,
'identifierSpace': 'http://rdf.freebase.com/ns/type.object.id',
'schemaSpace': 'http://rdf.freebase.com/ns/type.object.id',
'view': {
'url': urlp
},
'preview': {
'url': urlp + '?preview=true',
'width': 600,
'height': 300
},
'suggest': {
'entity': {
'service_url': domain,
'service_path': '/api/2/datasets/' + dataset.name + '/suggest'
}
},
'defaultTypes': [{
'name': dataset.label,
'id': '/' + dataset.name
}]
}
return jsonify(meta)
def handle_invalid(exc):
body = {
'status': 400,
'name': 'Invalid Data',
'description': unicode(exc),
'errors': exc.unpack_errors()
}
return jsonify(body, status=400)
def handle_invalid(exc):
body = {
'status': 400,
'name': 'Invalid Data',
'description': unicode(exc),
'errors': exc.unpack_errors()
}
return jsonify(body, status=400)
def handle_exceptions(exc):
message = exc.get_description(request.environ)
message = message.replace('<p>', '').replace('</p>', '')
body = {
'status': exc.code,
'name': exc.name,
'message': message
}
return jsonify(body, status=exc.code,
headers=exc.get_headers(request.environ))
def upload(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
file_ = request.files.get('file')
if not file_ or not file_.filename:
err = {'file': "You need to upload a file"}
raise Invalid("No file.", None, None, error_dict=err)
upload = Upload.create(dataset, request.account, file_)
db.session.commit()
return jsonify(upload)
def get_authz():
permissions = {}
dataset_name = request.args.get('dataset')
if dataset_name is not None:
dataset = Dataset.find(dataset_name)
permissions[dataset_name] = {
'view': True,
'edit': authz.dataset_edit(dataset),
'manage': authz.dataset_manage(dataset)
}
return jsonify(permissions)
def status():
return jsonify({
'logged_in':
authz.logged_in(),
'api_key':
request.account.api_key if authz.logged_in() else None,
'account':
request.account,
'base_url':
url_for('index', _external=True)
})
def get_authz():
permissions = {}
dataset_name = request.args.get('dataset')
if dataset_name is not None:
dataset = Dataset.find(dataset_name)
permissions[dataset_name] = {
'view': True,
'edit': authz.dataset_edit(dataset),
'manage': authz.dataset_manage(dataset)
}
return jsonify(permissions)
def reconcile_index(dataset):
domain = url_for("index", _external=True).strip("/")
urlp = domain + "/entities/{{id}}"
meta = {
"name": "nomenklatura: %s" % dataset.label,
"identifierSpace": "http://rdf.freebase.com/ns/type.object.id",
"schemaSpace": "http://rdf.freebase.com/ns/type.object.id",
"view": {"url": urlp},
"preview": {"url": urlp + "?preview=true", "width": 600, "height": 300},
"suggest": {"entity": {"service_url": domain, "service_path": "/api/2/reconcile/" + dataset.name + "/suggest"}},
"defaultTypes": [{"name": dataset.label, "id": "/" + dataset.name}],
}
return jsonify(meta)
def process(dataset, id):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
upload = Upload.find(dataset, id)
mapping = request_data()
mapping['reviewed'] = mapping.get('reviewed') or False
mapping['columns'] = mapping.get('columns', {})
fields = mapping['columns'].values()
for header in mapping['columns'].keys():
if header not in upload.tab.headers:
raise Invalid("Invalid header: %s" % header, None, None)
if 'name' not in fields and 'id' not in fields:
raise Invalid("You have not selected a field that definies entity names.", None, None)
import_upload.delay(upload.id, request.account.id, mapping)
return jsonify({'status': 'Loading data...'})
def reconcile_index(dataset):
domain = url_for('index', _external=True).strip('/')
urlp = domain + '/entities/{{id}}'
meta = {
'name': 'nomenklatura: %s' % dataset.label,
'identifierSpace': 'http://rdf.freebase.com/ns/type.object.id',
'schemaSpace': 'http://rdf.freebase.com/ns/type.object.id',
'view': {'url': urlp},
'preview': {
'url': urlp + '?preview=true',
'width': 600,
'height': 300
},
'suggest': {
'entity': {
'service_url': domain,
'service_path': '/api/2/datasets/' + dataset.name + '/suggest'
}
},
'defaultTypes': [{'name': dataset.label, 'id': '/' + dataset.name}]
}
return jsonify(meta)
def query_pager(q, paginate=True, serializer=lambda x: x, **kw):
limit = get_limit()
offset = get_offset()
if paginate:
results = q.offset(offset).limit(limit)
else:
results = q
url = url_for(request.endpoint, _external=True, **kw)
count = q.count()
data = {
'count': count,
'limit': limit,
'offset': offset,
'format': url + args('LIMIT', 'OFFSET'),
'previous': prev_url(url, count, offset, limit),
'next': next_url(url, count, offset, limit),
'results': map(serializer, results)
}
response = jsonify(data, refs=True)
if data['next']:
response.headers.add_header('Link', '<%s>; rel=next' % data['next'])
if data['previous']:
response.headers.add_header('Link', '<%s>; rel=previous' % data['previous'])
return response
def handle_exceptions(exc):
message = exc.get_description(request.environ)
message = message.replace('<p>', '').replace('</p>', '')
body = {'status': exc.code, 'name': exc.name, 'message': message}
headers = exc.get_headers(request.environ)
return jsonify(body, status=exc.code, headers=headers)
def view(dataset, id):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
upload = Upload.find(dataset, id)
return jsonify(upload)
def view(id):
entity = Entity.by_id(id)
return jsonify(entity)
def view(dataset):
dataset = Dataset.find(dataset)
return jsonify(dataset)
def view(id):
entity = object_or_404(Entity.by_id(id))
return jsonify(entity)
def by_name(dataset):
dataset = Dataset.find(dataset)
name = request.args.get('name')
entity = object_or_404(Entity.by_name(dataset, name))
return jsonify(entity)
def view(dataset):
dataset = Dataset.find(dataset)
return jsonify(dataset)
def status():
return jsonify({
'logged_in': authz.logged_in(),
'api_key': request.account.api_key if authz.logged_in() else None,
'account': request.account
})
def by_name(dataset):
dataset = Dataset.find(dataset)
name = request.args.get('name')
entity = object_or_404(Entity.by_name(dataset, name))
return jsonify(entity)
def view(id):
entity = object_or_404(Entity.by_id(id))
return jsonify(entity)
def attributes(dataset):
dataset = Dataset.find(dataset)
return jsonify({'attributes': attribute_keys(dataset)})
def handle_exceptions(exc):
message = exc.get_description(request.environ)
message = message.replace("<p>", "").replace("</p>", "")
body = {"status": exc.code, "name": exc.name, "message": message}
headers = exc.get_headers(request.environ)
return jsonify(body, status=exc.code, headers=headers)
def handle_invalid(exc):
body = {"status": 400, "name": "Invalid Data", "description": unicode(exc), "errors": exc.unpack_errors()}
return jsonify(body, status=400)