def reconcile(dataset=None):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
if dataset is not None:
dataset = Dataset.by_name(dataset)
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(dataset, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(dataset, q)
return jsonify(queries)
else:
return reconcile_index(dataset)
def lookup(dataset):
dataset = Dataset.find(dataset)
readonly = validators.StringBool(if_empty=False, if_missing=False)\
.to_python(request.args.get('readonly'))
readonly = readonly if authz.logged_in() else True
data = request_content()
if response_format() != 'json':
return Response("Not implemented!", status=400)
try:
alias = Alias.lookup(dataset, data, request.account,
readonly=readonly)
if alias is None:
return jsonify({
'is_matched': False,
'entity': None,
'name': data.get('name'),
'dataset': dataset.name
}, status=404)
if isinstance(alias, Entity):
return jsonify({
'is_matched': True,
'entity': alias,
'name': data.get('name'),
'dataset': dataset.name
}, status=200)
db.session.commit()
status = 200 if alias.is_matched else 404
status = 418 if alias.is_invalid else status
return jsonify(alias, status=status)
except Invalid, inv:
return handle_invalid(inv, index, data=data,
args=[dataset.name])
def reconcile(dataset=None):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
if dataset is not None:
dataset = Dataset.by_name(dataset)
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(dataset, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(dataset, q)
return jsonify(queries)
else:
return reconcile_index(dataset)
def view(dataset, entity):
dataset = Dataset.find(dataset)
entity = Entity.find(dataset, entity)
print entity.data
format = response_format()
if format == 'json':
return jsonify(entity)
query = request.args.get('query', '').strip().lower()
choices = match_op(entity.name, dataset)
choices = filter(lambda (c,e,s): e != entity.id, choices)
if len(query):
choices = filter(lambda (c,e,s): query in Entity.find(dataset,e).name.lower(),
choices)
# THIS is very inefficient - rather do this
# differently
pager = Pager(choices, '.view', dataset=dataset.name,
entity=entity.id, limit=10)
# HACK: Fetch only the entities on the selected page.
entities = Entity.id_map(dataset, map(lambda (c,v,s): v,
pager.query[pager.offset:pager.offset+pager.limit]))
for i, (c,e,s) in enumerate(pager.query):
if e in entities:
pager.query[i] = (c, entities.get(e), s)
return render_template('entity/view.html', dataset=dataset,
entity=entity, entities=pager, query=query)
def view(dataset, value):
dataset = Dataset.find(dataset)
value = Value.find(dataset, value)
format = response_format()
if format == 'json':
return jsonify(value)
query = request.args.get('query', '').strip().lower()
choices = match_op(value.value, dataset)
choices = filter(lambda (c,v,s): v != value.id, choices)
if len(query):
choices = filter(lambda (c,v,s): query in Value.find(dataset,v).value.lower(),
choices)
# THIS is very inefficient - rather do this
# differently
pager = Pager(choices, '.view', dataset=dataset.name,
value=value.id, limit=10)
# HACK: Fetch only the values on the selected page.
value_objs = Value.id_map(dataset, map(lambda (c,v,s): v,
pager.query[pager.offset:pager.offset+pager.limit]))
for i, (c,v,s) in enumerate(pager.query):
if v in value_objs:
pager.query[i] = (c, value_objs.get(v), s)
return render_template('value/view.html', dataset=dataset,
value=value, values=pager, query=query)
def view(dataset, link):
dataset = Dataset.find(dataset)
link = Link.find(dataset, link)
format = response_format()
if format == 'json':
return jsonify(link)
return "Not implemented!"
def reconcile_index(dataset):
domain = url_for('index', _external=True).strip('/')
urlp = domain + '{{id}}'
meta = {
'name': 'nomenklatura',
'identifierSpace': 'http://rdf.freebase.com/ns/type.object.id',
'schemaSpace': 'http://rdf.freebase.com/ns/type.object.id',
'view': {
'url': urlp
},
'preview': {
'url': urlp + '?preview=true',
'width': 600,
'height': 300
}
}
if dataset is not None:
meta['name'] = dataset.label
meta['suggest'] = {
'entity': {
'service_url': domain,
'service_path': '/' + dataset.name + '/suggest',
'flyout_service_path': '/flyout'
}
}
meta['defaultTypes'] = [{
'name': dataset.label,
'id': '/' + dataset.name
}]
else:
meta['defaultTypes'] = [{
'name': d.label,
'id': '/' + d.name
} for d in Dataset.all()]
return jsonify(meta)
def suggest(dataset):
"""
Suggest API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/SuggestApi
"""
try:
start = int(request.args.get('start', 0))
limit = int(request.args.get('limit', 20))
except:
raise BadRequest('Invalid result range!')
dataset = type_to_dataset(dataset)
query = request.args.get('prefix', '').strip()
results = prefix_search(query, dataset)[start:start + limit]
entities = Entity.id_map(dataset, map(lambda (c, v): v, results))
matches = []
for candidate, entity_id in results:
entity = entities[entity_id]
matches.append({
'name':
entity.name,
'n:type': {
'id': '/' + dataset.name,
'name': dataset.label
},
'id':
url_for('entity.view', dataset=dataset.name, entity=entity_id)
})
return jsonify({
"code": "/api/status/ok",
"status": "200 OK",
"prefix": query,
"result": matches
})
def suggest(dataset):
"""
Suggest API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/SuggestApi
"""
try:
start = int(request.args.get('start', 0))
limit = int(request.args.get('limit', 20))
except:
raise BadRequest('Invalid result range!')
dataset = type_to_dataset(dataset)
query = request.args.get('prefix', '').strip()
results = prefix_search(query, dataset)[start:start+limit]
entities = Entity.id_map(dataset, map(lambda (c,v): v, results))
matches = []
for candidate, entity_id in results:
entity = entities[entity_id]
matches.append({
'name': entity.name,
'n:type': {
'id': '/' + dataset.name,
'name': dataset.label
},
'id': url_for('entity.view', dataset=dataset.name, entity=entity_id)
})
return jsonify({
"code" : "/api/status/ok",
"status" : "200 OK",
"prefix" : query,
"result" : matches
})
def reconcile_index(dataset):
domain = url_for('index', _external=True).strip('/')
urlp = domain + '{{id}}'
meta = {
'name': 'nomenklatura',
'identifierSpace': 'http://rdf.freebase.com/ns/type.object.id',
'schemaSpace': 'http://rdf.freebase.com/ns/type.object.id',
'view': {'url': urlp},
'preview': {
'url': urlp + '?preview=true',
'width': 600,
'height': 300
}
}
if dataset is not None:
meta['name'] = dataset.label
meta['suggest'] = {
'entity': {
'service_url': domain,
'service_path': '/' + dataset.name + '/suggest',
'flyout_service_path': '/flyout'
}
}
meta['defaultTypes'] = [{'name': dataset.label, 'id': '/' + dataset.name}]
else:
meta['defaultTypes'] = [{'name': d.label, 'id': '/' + d.name} for d in Dataset.all()]
return jsonify(meta)
def handle_exceptions(exc):
""" Re-format exceptions to JSON if accept requires that. """
format = response_format()
if format == "json":
body = {"status": exc.code, "name": exc.name, "message": exc.get_description(request.environ)}
return jsonify(body, status=exc.code, headers=exc.get_headers(request.environ))
return exc
def index(dataset, format='json'):
dataset = Dataset.find(dataset)
q = Alias.all(dataset, eager=True)
if format == 'csv':
fn = csv_filename(dataset, 'aliases')
headers = {'Content-Disposition': 'attachment; filename=' + fn}
return csvify(q, headers=headers)
return jsonify(q)
def index(dataset, format='json'):
dataset = Dataset.find(dataset)
q = Entity.all(dataset, eager=True)
if format == 'csv':
fn = csv_filename(dataset, 'entities')
headers = {
'Content-Disposition': 'attachment; filename=' + fn
}
return csvify(q, headers=headers)
return jsonify(q)
def lookup(dataset):
dataset = Dataset.find(dataset)
readonly = validators.StringBool(if_empty=False, if_missing=False)\
.to_python(request.args.get('readonly'))
readonly = readonly if authz.logged_in() else True
data = request_content()
if response_format() != 'json':
return Response("Not implemented!", status=400)
try:
link = Link.lookup(dataset, data, request.account,
readonly=readonly)
if link is None:
return jsonify({
'is_matched': False,
'value': None,
'key': data.get('key'),
'dataset': dataset.name
}, status=404)
if isinstance(link, Value):
add_candidate_to_cache(dataset, data.get('key'), link.id)
return jsonify({
'is_matched': True,
'value': link,
'key': data.get('key'),
'dataset': dataset.name
}, status=200)
if link.value:
add_candidate_to_cache(dataset, link.key, link.value.id)
db.session.commit()
status = 200 if link.is_matched else 404
status = 418 if link.is_invalid else status
return jsonify(link, status=status)
except Invalid, inv:
return handle_invalid(inv, index, data=data,
args=[dataset.name])
def handle_exceptions(exc):
""" Re-format exceptions to JSON if accept requires that. """
format = response_format()
if format == 'json':
body = {
'status': exc.code,
'name': exc.name,
'message': exc.get_description(request.environ)
}
return jsonify(body,
status=exc.code,
headers=exc.get_headers(request.environ))
return exc
def lookup(dataset):
dataset = Dataset.find(dataset)
readonly = validators.StringBool(if_empty=False, if_missing=False)\
.to_python(request.args.get('readonly'))
readonly = readonly if authz.logged_in() else True
data = request_content()
if response_format() != 'json':
return Response("Not implemented!", status=400)
try:
alias = Alias.lookup(dataset, data, request.account, readonly=readonly)
if alias is None:
return jsonify(
{
'is_matched': False,
'entity': None,
'name': data.get('name'),
'dataset': dataset.name
},
status=404)
if isinstance(alias, Entity):
return jsonify(
{
'is_matched': True,
'entity': alias,
'name': data.get('name'),
'dataset': dataset.name
},
status=200)
db.session.commit()
status = 200 if alias.is_matched else 404
status = 418 if alias.is_invalid else status
return jsonify(alias, status=status)
except Invalid, inv:
return handle_invalid(inv, index, data=data, args=[dataset.name])
def handle_invalid(exc, html_func, data=None, args=()):
format = response_format()
if format == 'json':
body = {'status': 400,
'name': 'Invalid Data',
'message': unicode(exc),
'errors': exc.unpack_errors()}
return jsonify(body, status=400)
elif format == 'html':
data = data if data is not None else request_content()
content = htmlfill.render(html_func(*args),
defaults=data,
errors=exc.unpack_errors())
return Response(content, status=400, mimetype='text/html')
return Response(repr(exc.unpack_errors()), status=400,
mimetype='text/plain')
def view(dataset):
dataset = Dataset.find(dataset)
format = response_format()
if format == 'json':
return jsonify(dataset)
unmatched = Alias.all_unmatched(dataset).count()
entities = Entity.all(dataset, query=request.args.get('query'))
pager = Pager(entities, '.view', dataset=dataset.name,
limit=10)
return render_template('dataset/view.html',
entities=pager,
num_entities=len(pager),
num_aliases=Alias.all(dataset).count(),
invalid=Alias.all_invalid(dataset).count(),
query=request.args.get('query', ''),
dataset=dataset, unmatched=unmatched)
def view(dataset):
dataset = Dataset.find(dataset)
format = response_format()
if format == 'json':
return jsonify(dataset)
unmatched = Alias.all_unmatched(dataset).count()
entities = Entity.all(dataset, query=request.args.get('query'))
pager = Pager(entities, '.view', dataset=dataset.name, limit=10)
return render_template('dataset/view.html',
entities=pager,
num_entities=len(pager),
num_aliases=Alias.all(dataset).count(),
invalid=Alias.all_invalid(dataset).count(),
query=request.args.get('query', ''),
dataset=dataset,
unmatched=unmatched)
def view(dataset, value):
dataset = Dataset.find(dataset)
value = Value.find(dataset, value)
format = response_format()
if format == 'json':
return jsonify(value)
query = request.args.get('query', '').strip().lower()
choices = match_op(value.value, dataset)
choices = filter(lambda (c,v,s): v != value, choices)
if len(query):
choices = filter(lambda (c,v,s): query in v.value.lower(),
choices)
pager = Pager(choices, '.view', dataset=dataset.name,
value=value.id, limit=10)
return render_template('value/view.html', dataset=dataset,
value=value, values=pager, query=query)
def view(dataset):
dataset = Dataset.find(dataset)
format = response_format()
if format == "json":
return jsonify(dataset)
unmatched = Link.all_unmatched(dataset).count()
values = Value.all(dataset, query=request.args.get("query"))
pager = Pager(values, ".view", dataset=dataset.name, limit=10)
return render_template(
"dataset/view.html",
values=pager,
num_values=len(pager),
num_links=Link.all(dataset).count(),
invalid=Link.all_invalid(dataset).count(),
query=request.args.get("query", ""),
dataset=dataset,
unmatched=unmatched,
)
def view(dataset):
dataset = Dataset.find(dataset)
format = response_format()
headers = {
'X-Dataset': dataset.name,
'Last-Modified': http_date(dataset.last_modified)
}
if format == 'json':
return jsonify(dataset, headers=headers)
unmatched = Alias.all_unmatched(dataset).count()
entities = Entity.all(dataset, query=request.args.get('query'))
pager = Pager(entities, '.view', dataset=dataset.name,
limit=10)
html = render_template('dataset/view.html',
entities=pager,
num_entities=len(pager),
num_aliases=Alias.all(dataset).count(),
invalid=Alias.all_invalid(dataset).count(),
query=request.args.get('query', ''),
dataset=dataset, unmatched=unmatched)
return Response(html, headers=headers)
def view(dataset, entity):
dataset = Dataset.find(dataset)
entity = Entity.find(dataset, entity)
print entity.data
format = response_format()
if format == 'json':
return jsonify(entity)
query = request.args.get('query', '').strip().lower()
choices = match_op(entity.name, dataset)
choices = filter(lambda (c, e, s): e != entity.id, choices)
if len(query):
choices = filter(
lambda (c, e, s): query in Entity.find(dataset, e).name.lower(),
choices)
# THIS is very inefficient - rather do this
# differently
pager = Pager(choices,
'.view',
dataset=dataset.name,
entity=entity.id,
limit=10)
# HACK: Fetch only the entities on the selected page.
entities = Entity.id_map(
dataset,
map(lambda (c, v, s): v,
pager.query[pager.offset:pager.offset + pager.limit]))
for i, (c, e, s) in enumerate(pager.query):
if e in entities:
pager.query[i] = (c, entities.get(e), s)
return render_template('entity/view.html',
dataset=dataset,
entity=entity,
entities=pager,
query=query)
def flyout():
return jsonify({'html': '<h3>%s</h3>' % request.args.get('id')})
def flyout():
return jsonify({'html': '<h3>%s</h3>' % request.args.get('id')})
def view(dataset, alias):
dataset = Dataset.find(dataset)
alias = Alias.find(dataset, alias)
#format = response_format()
#if format == 'json':
return jsonify(alias)
'name': alias.name,
'query': request.args.get('query', ''),
'random': random
})
@section.route('/<dataset>/aliases/<alias>/match', methods=['POST'])
def match_save(dataset, alias):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
alias = Alias.find(dataset, alias)
random = request.form.get('random') == 'True'
data = request_content()
try:
alias.match(dataset, data, request.account)
db.session.commit()
except Invalid, inv:
return handle_invalid(inv,
match,
data=data,
args=[dataset.name, alias.id, random])
flash("Matched: %s" % alias.name, "success")
format = response_format()
if format == 'json':
return jsonify(alias)
if random:
return match_random(dataset.name)
else:
return match(dataset.name, alias.id)
def index(dataset):
dataset = Dataset.find(dataset)
format = response_format()
if format == 'json':
return jsonify(Value.all(dataset))
return "Not implemented!"
def index():
format = response_format()
if format == 'json':
return jsonify(Dataset.all())
return "Not implemented!"
def index():
format = response_format()
if format == 'json':
return jsonify(Dataset.all())
return "Not implemented!"
def view(dataset, alias):
dataset = Dataset.find(dataset)
alias = Alias.find(dataset, alias)
#format = response_format()
#if format == 'json':
return jsonify(alias)
choice = 'INVALID' if s <= 50 else e.id
return htmlfill.render(html, force_defaults=False,
defaults={'choice': choice,
'name': alias.name,
'query': request.args.get('query', ''),
'random': random})
@section.route('/<dataset>/aliases/<alias>/match', methods=['POST'])
def match_save(dataset, alias):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
alias = Alias.find(dataset, alias)
random = request.form.get('random')=='True'
data = request_content()
try:
alias.match(dataset, data, request.account)
db.session.commit()
except Invalid, inv:
return handle_invalid(inv, match, data=data,
args=[dataset.name, alias.id, random])
flash("Matched: %s" % alias.name, "success")
format = response_format()
if format == 'json':
return jsonify(alias)
if random:
return match_random(dataset.name)
else:
return match(dataset.name, alias.id)
choice = choices[0][1].id if choice is None else choice
return htmlfill.render(html, force_defaults=False,
defaults={'choice': choice,
'value': link.key,
'query': request.args.get('query', ''),
'random': random})
@section.route('/<dataset>/links/<link>/match', methods=['POST'])
def match_save(dataset, link):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
link = Link.find(dataset, link)
random = request.form.get('random')=='True'
data = request_content()
try:
link.match(dataset, data, request.account)
db.session.commit()
except Invalid, inv:
return handle_invalid(inv, match, data=data,
args=[dataset.name, link.id, random])
flash("Matched: %s" % link.key, "success")
format = response_format()
if format == 'json':
return jsonify(link)
if random:
return match_random(dataset.name)
else:
return match(dataset.name, link.id)
def index(dataset):
dataset = Dataset.find(dataset)
format = response_format()
if format == 'json':
return jsonify(Link.all(dataset, eager=True))
return "Not implemented!"
