def import_upload(dataset_name, id, account_id,
entity_col, alias_col):
dataset = Dataset.find(dataset_name)
account = Account.by_id(account_id)
metadata, row_set = parse_upload(dataset, id)
headers = detect_headers(row_set)
for row in row_set:
data = dict([(c.column, c.value) for c in row])
entity = data.pop(entity_col) if entity_col else None
alias = data.pop(alias_col) if alias_col else None
if alias_col and alias is not None and len(alias) and alias != entity:
d = {'name': alias, 'data': data}
alias_obj = Alias.lookup(dataset, d, account,
match_entity=False)
data = {}
if entity_col and entity is not None and len(entity):
d = {'name': entity, 'data': data}
entity_obj = Entity.by_name(dataset, entity)
if entity_obj is None:
entity_obj = Entity.create(dataset, d, account)
entity_obj.data = data
if alias_col and entity_col:
alias_obj.match(dataset, {'choice': entity_obj.id}, account)
db.session.commit()
flush_cache(dataset)
def match(dataset, alias, random=False):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
alias = Alias.find(dataset, alias)
random = random or request.args.get('random')=='True'
choices = match_op(alias.name, dataset,
query=request.args.get('query'))
pager = Pager(choices, '.match',
dataset=dataset.name, alias=alias.id,
limit=10)
# HACK: Fetch only the entities on the selected page.
entities = Entity.id_map(dataset, map(lambda (c,e,s): e,
pager.query[pager.offset:pager.offset+pager.limit]))
for i, (c,e,s) in enumerate(pager.query):
if e in entities:
pager.query[i] = (c, entities.get(e), s)
html = render_template('alias/match.html',
dataset=dataset, alias=alias, choices=pager,
random=random)
choice = 'INVALID' if alias.is_invalid else alias.entity_id
if len(choices) and choice is None:
c, e, s = choices[0]
choice = 'INVALID' if s <= 50 else e.id
return htmlfill.render(html, force_defaults=False,
defaults={'choice': choice,
'name': alias.name,
'query': request.args.get('query', ''),
'random': random})
def match(dataset, link, random=False):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
link = Link.find(dataset, link)
random = random or request.args.get('random')=='True'
choices = match_op(link.key, dataset,
query=request.args.get('query'))
pager = Pager(choices, '.match',
dataset=dataset.name, link=link.id,
limit=10)
# HACK: Fetch only the values on the selected page.
value_objs = Value.id_map(dataset, map(lambda (c,v,s): v,
pager.query[pager.offset:pager.offset+pager.limit]))
for i, (c,v,s) in enumerate(pager.query):
if v in value_objs:
pager.query[i] = (c, value_objs.get(v), s)
html = render_template('link/match.html',
dataset=dataset, link=link, choices=pager,
random=random)
choice = 'INVALID' if link.is_invalid else link.value_id
if len(choices) and choice is None:
c, v, s = choices[0]
choice = 'INVALID' if s <= 50 else v
return htmlfill.render(html, force_defaults=False,
defaults={'choice': choice,
'value': link.key,
'query': request.args.get('query', ''),
'random': random})
def reconcile_index(dataset):
domain = url_for('index', _external=True).strip('/')
urlp = domain + '{{id}}'
meta = {
'name': 'nomenklatura',
'identifierSpace': 'http://rdf.freebase.com/ns/type.object.id',
'schemaSpace': 'http://rdf.freebase.com/ns/type.object.id',
'view': {
'url': urlp
},
'preview': {
'url': urlp + '?preview=true',
'width': 600,
'height': 300
}
}
if dataset is not None:
meta['name'] = dataset.label
meta['suggest'] = {
'entity': {
'service_url': domain,
'service_path': '/' + dataset.name + '/suggest',
'flyout_service_path': '/flyout'
}
}
meta['defaultTypes'] = [{
'name': dataset.label,
'id': '/' + dataset.name
}]
else:
meta['defaultTypes'] = [{
'name': d.label,
'id': '/' + d.name
} for d in Dataset.all()]
return jsonify(meta)
def create():
authz.require(authz.dataset_create())
if app.config.get('DATASET_CREATION_DISABLED'):
raise Forbidden("Sorry, dataset creation is disabled")
dataset = Dataset.create(request_data(), request.account)
db.session.commit()
return redirect(url_for('.view', dataset=dataset.name))
def view(dataset, value):
dataset = Dataset.find(dataset)
value = Value.find(dataset, value)
format = response_format()
if format == 'json':
return jsonify(value)
query = request.args.get('query', '').strip().lower()
choices = match_op(value.value, dataset)
choices = filter(lambda (c,v,s): v != value.id, choices)
if len(query):
choices = filter(lambda (c,v,s): query in Value.find(dataset,v).value.lower(),
choices)
# THIS is very inefficient - rather do this
# differently
pager = Pager(choices, '.view', dataset=dataset.name,
value=value.id, limit=10)
# HACK: Fetch only the values on the selected page.
value_objs = Value.id_map(dataset, map(lambda (c,v,s): v,
pager.query[pager.offset:pager.offset+pager.limit]))
for i, (c,v,s) in enumerate(pager.query):
if v in value_objs:
pager.query[i] = (c, value_objs.get(v), s)
return render_template('value/view.html', dataset=dataset,
value=value, values=pager, query=query)
def suggest(dataset):
"""
Suggest API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/SuggestApi
"""
dataset = Dataset.by_name(dataset)
entities = Entity.all().filter(Entity.invalid!=True)
query = request.args.get('prefix', '').strip()
entities = entities.filter(Entity.name.ilike('%s%%' % query))
entities = entities.offset(get_offset(field='start'))
entities = entities.limit(get_limit(default=20))
matches = []
for entity in entities:
matches.append({
'name': entity.name,
'n:type': {
'id': '/' + dataset.name,
'name': dataset.label
},
'id': entity.id
})
return jsonify({
"code" : "/api/status/ok",
"status" : "200 OK",
"prefix" : query,
"result" : matches
})
def reconcile_index(dataset):
domain = url_for('index', _external=True).strip('/')
urlp = domain + '{{id}}'
meta = {
'name': 'nomenklatura',
'identifierSpace': 'http://rdf.freebase.com/ns/type.object.id',
'schemaSpace': 'http://rdf.freebase.com/ns/type.object.id',
'view': {'url': urlp},
'preview': {
'url': urlp + '?preview=true',
'width': 600,
'height': 300
}
}
if dataset is not None:
meta['name'] = dataset.label
meta['suggest'] = {
'entity': {
'service_url': domain,
'service_path': '/' + dataset.name + '/suggest',
'flyout_service_path': '/flyout'
}
}
meta['defaultTypes'] = [{'name': dataset.label, 'id': '/' + dataset.name}]
else:
meta['defaultTypes'] = [{'name': d.label, 'id': '/' + d.name} for d in Dataset.all()]
return jsonify(meta)
def reconcile(dataset):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
dataset = Dataset.by_name(dataset)
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(dataset, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(dataset, q)
return jsonify(queries)
else:
return reconcile_index(dataset)
def import_upload(dataset_name, sig, account_id,
value_col, link_col):
dataset = Dataset.find(dataset_name)
account = Account.by_id(account_id)
metadata, row_set = parse_upload(dataset, sig)
headers = detect_headers(row_set)
for row in row_set:
data = dict([(c.column, c.value) for c in row])
value = data.pop(value_col) if value_col else None
link = data.pop(link_col) if link_col else None
if link_col:
d = {'key': link, 'data': data}
link_obj = Link.lookup(dataset, d, account,
match_value=False)
data = {}
if value_col:
d = {'value': value, 'data': data}
value_obj = Value.by_value(dataset, value)
if value_obj is None:
value_obj = Value.create(dataset,
d, account)
value_obj.data = data
if link_col and value_col:
link_obj.match(dataset, {'choice': value_obj.id},
account)
db.session.commit()
def review(dataset):
entities = Entity.all()
dataset = Dataset.find(dataset)
entities = entities.filter_by(dataset=dataset)
entities = entities.filter(Entity.reviewed==False)
entities = entities.offset(randint(0, entities.count()-1))
return jsonify(entities.first())
def view(dataset, entity):
dataset = Dataset.find(dataset)
entity = Entity.find(dataset, entity)
print entity.data
format = response_format()
if format == 'json':
return jsonify(entity)
query = request.args.get('query', '').strip().lower()
choices = match_op(entity.name, dataset)
choices = filter(lambda (c,e,s): e != entity.id, choices)
if len(query):
choices = filter(lambda (c,e,s): query in Entity.find(dataset,e).name.lower(),
choices)
# THIS is very inefficient - rather do this
# differently
pager = Pager(choices, '.view', dataset=dataset.name,
entity=entity.id, limit=10)
# HACK: Fetch only the entities on the selected page.
entities = Entity.id_map(dataset, map(lambda (c,v,s): v,
pager.query[pager.offset:pager.offset+pager.limit]))
for i, (c,e,s) in enumerate(pager.query):
if e in entities:
pager.query[i] = (c, entities.get(e), s)
return render_template('entity/view.html', dataset=dataset,
entity=entity, entities=pager, query=query)
def lookup(dataset):
dataset = Dataset.find(dataset)
readonly = validators.StringBool(if_empty=False, if_missing=False)\
.to_python(request.args.get('readonly'))
readonly = readonly if authz.logged_in() else True
data = request_content()
if response_format() != 'json':
return Response("Not implemented!", status=400)
try:
alias = Alias.lookup(dataset, data, request.account,
readonly=readonly)
if alias is None:
return jsonify({
'is_matched': False,
'entity': None,
'name': data.get('name'),
'dataset': dataset.name
}, status=404)
if isinstance(alias, Entity):
return jsonify({
'is_matched': True,
'entity': alias,
'name': data.get('name'),
'dataset': dataset.name
}, status=200)
db.session.commit()
status = 200 if alias.is_matched else 404
status = 418 if alias.is_invalid else status
return jsonify(alias, status=status)
except Invalid, inv:
return handle_invalid(inv, index, data=data,
args=[dataset.name])
def view(dataset, link):
dataset = Dataset.find(dataset)
link = Link.find(dataset, link)
format = response_format()
if format == 'json':
return jsonify(link)
return "Not implemented!"
def create():
data = request_data()
dataset = Dataset.from_form(data)
authz.require(authz.dataset_edit(dataset))
entity = Entity.create(dataset, data, request.account)
db.session.commit()
return redirect(url_for('.view', id=entity.id))
def edit(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_manage(dataset))
html = render_template('dataset/edit.html',
dataset=dataset,
algorithms=get_algorithms())
return htmlfill.render(html, defaults=dataset.as_dict())
def create():
data = request_data()
dataset = Dataset.from_form(data)
authz.require(authz.dataset_edit(dataset))
entity = Entity.create(dataset, data, request.account)
db.session.commit()
return redirect(url_for('.view', id=entity.id))
def edit(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_manage(dataset))
html = render_template('dataset/edit.html',
dataset=dataset,
algorithms=get_algorithms())
return htmlfill.render(html, defaults=dataset.as_dict())
def reconcile(dataset=None):
"""
Reconciliation API, emulates Google Refine API. See:
http://code.google.com/p/google-refine/wiki/ReconciliationServiceApi
"""
if dataset is not None:
dataset = Dataset.by_name(dataset)
# TODO: Add proper support for types and namespacing.
data = request.args.copy()
data.update(request.form.copy())
if 'query' in data:
# single
q = data.get('query')
if q.startswith('{'):
try:
q = json.loads(q)
except ValueError:
raise BadRequest()
else:
q = data
return jsonify(reconcile_op(dataset, q))
elif 'queries' in data:
# multiple requests in one query
qs = data.get('queries')
try:
qs = json.loads(qs)
except ValueError:
raise BadRequest()
queries = {}
for k, q in qs.items():
queries[k] = reconcile_op(dataset, q)
return jsonify(queries)
else:
return reconcile_index(dataset)
def set_template_globals():
return {
"datasets": Dataset.all(),
"authz": authz,
"avatar_url": session.get("avatar_url", ""),
"logged_in": request.account is not None,
"login": request.account.login if request.account else None,
}
def set_template_globals():
return {
'datasets': Dataset.all(),
'authz': authz,
'avatar_url': session.get('avatar_url', ''),
'logged_in': request.account is not None,
'login': request.account.login if request.account else None
}
def index(dataset, format='json'):
dataset = Dataset.find(dataset)
q = Alias.all(dataset, eager=True)
if format == 'csv':
fn = csv_filename(dataset, 'aliases')
headers = {'Content-Disposition': 'attachment; filename=' + fn}
return csvify(q, headers=headers)
return jsonify(q)
def match():
dataset_arg = request.args.get('dataset')
dataset = Dataset.find(dataset_arg)
matches = find_matches(dataset,
request.args.get('name'),
filter=request.args.get('filter'),
exclude=arg_int('exclude'))
return query_pager(matches)
def set_template_globals():
return {
'datasets': Dataset.all(),
'authz': authz,
'avatar_url': session.get('avatar_url', ''),
'logged_in': request.account is not None,
'login': request.account.login if request.account else None
}
def match():
dataset_arg = request.args.get('dataset')
dataset = Dataset.find(dataset_arg)
matches = find_matches(dataset,
request.args.get('name'),
filter=request.args.get('filter'),
exclude=arg_int('exclude'))
return query_pager(matches)
def match():
dataset_arg = request.args.get('dataset')
dataset = Dataset.find(dataset_arg)
matches = find_matches(dataset,
request.args.get('name'),
filter=request.args.get('filter'),
exclude=arg_int('exclude'))
pager = Pager(matches)
return jsonify(pager.to_dict())
def create():
authz.require(authz.dataset_create())
data = request_content()
try:
dataset = Dataset.create(data, request.account)
db.session.commit()
return redirect(url_for('.view', dataset=dataset.name))
except Invalid, inv:
return handle_invalid(inv, new, data=data)
def create():
authz.require(authz.dataset_create())
data = request_content()
try:
dataset = Dataset.create(data, request.account)
db.session.commit()
return redirect(url_for('.view', dataset=dataset.name))
except Invalid, inv:
return handle_invalid(inv, new, data=data)
def match():
dataset_arg = request.args.get('dataset')
dataset = Dataset.find(dataset_arg)
matches = find_matches(dataset,
request.args.get('name'),
filter=request.args.get('filter'),
exclude=arg_int('exclude'))
pager = Pager(matches)
return jsonify(pager.to_dict())
def upload(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
file_ = request.files.get('file')
if not file_ or not file_.filename:
inv = Invalid("No file.", None, None,
error_dict={'file': "You need to upload a file"})
raise inv
upload = upload_file(dataset, file_, request.account)
return redirect(url_for('.map', dataset=dataset.name, id=upload.id))
def index(dataset, format='json'):
dataset = Dataset.find(dataset)
q = Entity.all(dataset, eager=True)
if format == 'csv':
fn = csv_filename(dataset, 'entities')
headers = {
'Content-Disposition': 'attachment; filename=' + fn
}
return csvify(q, headers=headers)
return jsonify(q)
def review(dataset):
entities = Entity.all()
dataset = Dataset.find(dataset)
entities = entities.filter_by(dataset=dataset)
entities = entities.filter(Entity.reviewed == False) # noqa
review_count = entities.count()
if review_count == 0:
return jsonify(None)
entities = entities.offset(randint(0, review_count - 1))
return jsonify(entities.first())
def upload(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
file_ = request.files.get('file')
if not file_ or not file_.filename:
err = {'file': "You need to upload a file"}
raise Invalid("No file.", None, None, error_dict=err)
upload = Upload.create(dataset, request.account, file_)
db.session.commit()
return jsonify(upload)
def match_random(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
aliases = Alias.all_unmatched(dataset)
count = aliases.count()
if count == 0:
return redirect(url_for('dataset.view', dataset=dataset.name))
alias = aliases.offset(randint(0, count - 1)).first()
return redirect(
url_for('.match', dataset=dataset.name, alias=alias.id, random=True))
def review(dataset):
entities = Entity.all()
dataset = Dataset.find(dataset)
entities = entities.filter_by(dataset=dataset)
entities = entities.filter(Entity.reviewed == False)
review_count = entities.count()
if review_count == 0:
return jsonify(None)
entities = entities.offset(randint(0, review_count - 1))
return jsonify(entities.first())
def update(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_manage(dataset))
data = request_content()
try:
dataset.update(data)
db.session.commit()
flash("Updated %s" % dataset.label, "success")
return redirect(url_for(".view", dataset=dataset.name))
except Invalid, inv:
return handle_invalid(inv, edit, args=[dataset.name], data=data)
def get_authz():
permissions = {}
dataset_name = request.args.get("dataset")
if dataset_name is not None:
dataset = Dataset.find(dataset_name)
permissions[dataset_name] = {
"view": True,
"edit": authz.dataset_edit(dataset),
"manage": authz.dataset_manage(dataset),
}
return jsonify(permissions)
def get_authz():
permissions = {}
dataset_name = request.args.get('dataset')
if dataset_name is not None:
dataset = Dataset.find(dataset_name)
permissions[dataset_name] = {
'view': True,
'edit': authz.dataset_edit(dataset),
'manage': authz.dataset_manage(dataset)
}
return jsonify(permissions)
def upload(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
file_ = request.files.get('file')
if not file_ or not file_.filename:
inv = Invalid("No file.", None, None,
error_dict={'file': "You need to upload a file"})
return handle_invalid(inv, form, data={},
args=[dataset.name])
sig = upload_file(dataset, file_)
return redirect(url_for('.map', dataset=dataset.name, sig=sig))
def get_authz():
permissions = {}
dataset_name = request.args.get('dataset')
if dataset_name is not None:
dataset = Dataset.find(dataset_name)
permissions[dataset_name] = {
'view': True,
'edit': authz.dataset_edit(dataset),
'manage': authz.dataset_manage(dataset)
}
return jsonify(permissions)
def match_random(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
aliases = Alias.all_unmatched(dataset)
count = aliases.count()
if count == 0:
return redirect(url_for('dataset.view',
dataset=dataset.name))
alias = aliases.offset(randint(0, count-1)).first()
return redirect(url_for('.match', dataset=dataset.name, alias=alias.id,
random=True))
def upload(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
file_ = request.files.get('file')
if not file_ or not file_.filename:
inv = Invalid("No file.",
None,
None,
error_dict={'file': "You need to upload a file"})
return handle_invalid(inv, form, data={}, args=[dataset.name])
sig = upload_file(dataset, file_)
return redirect(url_for('.map', dataset=dataset.name, sig=sig))
def update(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_manage(dataset))
data = request_content()
try:
flush_cache(dataset)
dataset.update(data)
db.session.commit()
flash("Updated %s" % dataset.label, 'success')
return redirect(url_for('.view', dataset=dataset.name))
except Invalid, inv:
return handle_invalid(inv, edit, args=[dataset.name], data=data)
def match_save(dataset, alias):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
alias = Alias.find(dataset, alias)
random = request.form.get('random')=='True'
data = request_content()
try:
alias.match(dataset, data, request.account)
db.session.commit()
except Invalid, inv:
return handle_invalid(inv, match, data=data,
args=[dataset.name, alias.id, random])
def submit(dataset, sig):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
data = request_content()
entity_col = data.get('entity') or None
alias_col = data.get('alias') or None
if not (entity_col or alias_col):
flash('You need to pick either a alias or entity column!', 'error')
return map(dataset.name, sig)
import_upload.delay(dataset.name, sig, request.account.id, entity_col,
alias_col)
flash('Loading data...', 'success')
return redirect(url_for('dataset.view', dataset=dataset.name))
def submit(dataset, id):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
data = request_content()
entity_col = data.get('entity') or None
alias_col = data.get('alias') or None
if not (entity_col or alias_col):
flash('You need to pick either a alias or entity column!', 'error')
return map(dataset.name, id)
import_upload.delay(dataset.name, id, request.account.id,
entity_col, alias_col)
flash('Loading data...', 'success')
return redirect(url_for('dataset.view', dataset=dataset.name))
def create(dataset):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
data = request_content()
try:
entity = Entity.create(dataset, data, request.account)
db.session.commit()
return redirect(
url_for('.view', dataset=dataset.name, entity=entity.id))
except Invalid, inv:
return handle_invalid(inv,
view_dataset,
data=data,
args=[dataset.name])
def match_save(dataset, alias):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
alias = Alias.find(dataset, alias)
random = request.form.get('random') == 'True'
data = request_content()
try:
alias.match(dataset, data, request.account)
db.session.commit()
except Invalid, inv:
return handle_invalid(inv,
match,
data=data,
args=[dataset.name, alias.id, random])
def update(dataset, entity):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
entity = Entity.find(dataset, entity)
data = request_content()
try:
entity.update(data, request.account)
db.session.commit()
flash("Updated %s" % entity.display_name, 'success')
return redirect(
url_for('.view', dataset=dataset.name, entity=entity.id))
except Invalid, inv:
return handle_invalid(inv,
view,
data=data,
args=[dataset.name, entity.id])
def view(dataset):
dataset = Dataset.find(dataset)
format = response_format()
if format == 'json':
return jsonify(dataset)
unmatched = Alias.all_unmatched(dataset).count()
entities = Entity.all(dataset, query=request.args.get('query'))
pager = Pager(entities, '.view', dataset=dataset.name, limit=10)
return render_template('dataset/view.html',
entities=pager,
num_entities=len(pager),
num_aliases=Alias.all(dataset).count(),
invalid=Alias.all_invalid(dataset).count(),
query=request.args.get('query', ''),
dataset=dataset,
unmatched=unmatched)
def process(dataset, id):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
upload = Upload.find(dataset, id)
mapping = request_data()
mapping['reviewed'] = mapping.get('reviewed') or False
mapping['columns'] = mapping.get('columns', {})
fields = mapping['columns'].values()
for header in mapping['columns'].keys():
if header not in upload.tab.headers:
raise Invalid("Invalid header: %s" % header, None, None)
if 'name' not in fields and 'id' not in fields:
raise Invalid("You have not selected a field that definies entity names.", None, None)
import_upload.delay(upload.id, request.account.id, mapping)
return jsonify({'status': 'Loading data...'})
def match(dataset, alias, random=False):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
alias = Alias.find(dataset, alias)
random = random or request.args.get('random') == 'True'
choices = match_op(alias.name, dataset, query=request.args.get('query'))
pager = Pager(choices,
'.match',
dataset=dataset.name,
alias=alias.id,
limit=10)
# HACK: Fetch only the entities on the selected page.
entities = Entity.id_map(
dataset,
map(lambda (c, e, s): e,
pager.query[pager.offset:pager.offset + pager.limit]))
for i, (c, e, s) in enumerate(pager.query):
if e in entities:
pager.query[i] = (c, entities.get(e), s)
html = render_template('alias/match.html',
dataset=dataset,
alias=alias,
choices=pager,
random=random)
choice = 'INVALID' if alias.is_invalid else alias.entity_id
if len(choices) and choice is None:
c, e, s = choices[0]
choice = 'INVALID' if s <= 50 else e.id
return htmlfill.render(html,
force_defaults=False,
defaults={
'choice': choice,
'name': alias.name,
'query': request.args.get('query', ''),
'random': random
})
def import_upload(dataset_name, sig, account_id, entity_col, alias_col):
dataset = Dataset.find(dataset_name)
account = Account.by_id(account_id)
metadata, row_set = parse_upload(dataset, sig)
headers = detect_headers(row_set)
for row in row_set:
data = dict([(c.column, c.value) for c in row])
entity = data.pop(entity_col) if entity_col else None
alias = data.pop(alias_col) if alias_col else None
if alias_col and alias is not None and len(alias) and alias != entity:
d = {'name': alias, 'data': data}
alias_obj = Alias.lookup(dataset, d, account, match_entity=False)
data = {}
if entity_col and entity is not None and len(entity):
d = {'name': entity, 'data': data}
entity_obj = Entity.by_name(dataset, entity)
if entity_obj is None:
entity_obj = Entity.create(dataset, d, account)
entity_obj.data = data
if alias_col and entity_col:
alias_obj.match(dataset, {'choice': entity_obj.id}, account)
db.session.commit()
flush_cache()
def index():
entities = Entity.all()
dataset_arg = request.args.get('dataset')
if dataset_arg is not None:
dataset = Dataset.find(dataset_arg)
entities = entities.filter_by(dataset=dataset)
filter_name = request.args.get('filter_name', '')
if len(filter_name):
query = '%' + filter_name + '%'
entities = entities.filter(Entity.name.ilike(query))
# TODO, other filters.
format = request.args.get('format', 'json').lower().strip()
if format == 'csv':
res = csvify(entities)
else:
res = query_pager(entities)
if arg_bool('download'):
fn = dataset_filename(dataset, format)
res.headers['Content-Disposition'] = 'attachment; filename=' + fn
return res
def lookup(dataset):
dataset = Dataset.find(dataset)
readonly = validators.StringBool(if_empty=False, if_missing=False)\
.to_python(request.args.get('readonly'))
readonly = readonly if authz.logged_in() else True
data = request_content()
if response_format() != 'json':
return Response("Not implemented!", status=400)
try:
alias = Alias.lookup(dataset, data, request.account, readonly=readonly)
if alias is None:
return jsonify(
{
'is_matched': False,
'entity': None,
'name': data.get('name'),
'dataset': dataset.name
},
status=404)
if isinstance(alias, Entity):
return jsonify(
{
'is_matched': True,
'entity': alias,
'name': data.get('name'),
'dataset': dataset.name
},
status=200)
db.session.commit()
status = 200 if alias.is_matched else 404
status = 418 if alias.is_invalid else status
return jsonify(alias, status=status)
except Invalid, inv:
return handle_invalid(inv, index, data=data, args=[dataset.name])
def view(dataset, entity):
dataset = Dataset.find(dataset)
entity = Entity.find(dataset, entity)
print entity.data
format = response_format()
if format == 'json':
return jsonify(entity)
query = request.args.get('query', '').strip().lower()
choices = match_op(entity.name, dataset)
choices = filter(lambda (c, e, s): e != entity.id, choices)
if len(query):
choices = filter(
lambda (c, e, s): query in Entity.find(dataset, e).name.lower(),
choices)
# THIS is very inefficient - rather do this
# differently
pager = Pager(choices,
'.view',
dataset=dataset.name,
entity=entity.id,
limit=10)
# HACK: Fetch only the entities on the selected page.
entities = Entity.id_map(
dataset,
map(lambda (c, v, s): v,
pager.query[pager.offset:pager.offset + pager.limit]))
for i, (c, e, s) in enumerate(pager.query):
if e in entities:
pager.query[i] = (c, entities.get(e), s)
return render_template('entity/view.html',
dataset=dataset,
entity=entity,
entities=pager,
query=query)
def view(dataset, id):
dataset = Dataset.find(dataset)
authz.require(authz.dataset_edit(dataset))
upload = Upload.find(dataset, id)
return jsonify(upload)
def view_by_name(dataset):
dataset = Dataset.find(dataset)
alias = Alias.by_name(dataset, request.args.get('name'))
if alias is None:
raise NotFound("No such alias: %s" % request.args.get('name'))
return view(dataset.name, alias.id)
