def import_upload(dataset_name, sig, account_id, value_col, link_col): dataset = Dataset.find(dataset_name) account = Account.by_id(account_id) metadata, row_set = parse_upload(dataset, sig) headers = detect_headers(row_set) for row in row_set: data = dict([(c.column, c.value) for c in row]) value = data.pop(value_col) if value_col else None link = data.pop(link_col) if link_col else None if link_col: d = {'key': link, 'data': data} link_obj = Link.lookup(dataset, d, account, match_value=False) data = {} if value_col: d = {'value': value, 'data': data} value_obj = Value.by_value(dataset, value) if value_obj is None: value_obj = Value.create(dataset, d, account) value_obj.data = data if link_col and value_col: link_obj.match(dataset, {'choice': value_obj.id}, account) db.session.commit()
def import_upload(dataset_name, id, account_id, entity_col, alias_col): dataset = Dataset.find(dataset_name) account = Account.by_id(account_id) metadata, row_set = parse_upload(dataset, id) headers = detect_headers(row_set) for row in row_set: data = dict([(c.column, c.value) for c in row]) entity = data.pop(entity_col) if entity_col else None alias = data.pop(alias_col) if alias_col else None if alias_col and alias is not None and len(alias) and alias != entity: d = {'name': alias, 'data': data} alias_obj = Alias.lookup(dataset, d, account, match_entity=False) data = {} if entity_col and entity is not None and len(entity): d = {'name': entity, 'data': data} entity_obj = Entity.by_name(dataset, entity) if entity_obj is None: entity_obj = Entity.create(dataset, d, account) entity_obj.data = data if alias_col and entity_col: alias_obj.match(dataset, {'choice': entity_obj.id}, account) db.session.commit() flush_cache(dataset)
def match(dataset, link, random=False): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) link = Link.find(dataset, link) random = random or request.args.get('random')=='True' choices = match_op(link.key, dataset, query=request.args.get('query')) pager = Pager(choices, '.match', dataset=dataset.name, link=link.id, limit=10) # HACK: Fetch only the values on the selected page. value_objs = Value.id_map(dataset, map(lambda (c,v,s): v, pager.query[pager.offset:pager.offset+pager.limit])) for i, (c,v,s) in enumerate(pager.query): if v in value_objs: pager.query[i] = (c, value_objs.get(v), s) html = render_template('link/match.html', dataset=dataset, link=link, choices=pager, random=random) choice = 'INVALID' if link.is_invalid else link.value_id if len(choices) and choice is None: c, v, s = choices[0] choice = 'INVALID' if s <= 50 else v return htmlfill.render(html, force_defaults=False, defaults={'choice': choice, 'value': link.key, 'query': request.args.get('query', ''), 'random': random})
def review(dataset): entities = Entity.all() dataset = Dataset.find(dataset) entities = entities.filter_by(dataset=dataset) entities = entities.filter(Entity.reviewed==False) entities = entities.offset(randint(0, entities.count()-1)) return jsonify(entities.first())
def lookup(dataset): dataset = Dataset.find(dataset) readonly = validators.StringBool(if_empty=False, if_missing=False)\ .to_python(request.args.get('readonly')) readonly = readonly if authz.logged_in() else True data = request_content() if response_format() != 'json': return Response("Not implemented!", status=400) try: alias = Alias.lookup(dataset, data, request.account, readonly=readonly) if alias is None: return jsonify({ 'is_matched': False, 'entity': None, 'name': data.get('name'), 'dataset': dataset.name }, status=404) if isinstance(alias, Entity): return jsonify({ 'is_matched': True, 'entity': alias, 'name': data.get('name'), 'dataset': dataset.name }, status=200) db.session.commit() status = 200 if alias.is_matched else 404 status = 418 if alias.is_invalid else status return jsonify(alias, status=status) except Invalid, inv: return handle_invalid(inv, index, data=data, args=[dataset.name])
def view(dataset, value): dataset = Dataset.find(dataset) value = Value.find(dataset, value) format = response_format() if format == 'json': return jsonify(value) query = request.args.get('query', '').strip().lower() choices = match_op(value.value, dataset) choices = filter(lambda (c,v,s): v != value.id, choices) if len(query): choices = filter(lambda (c,v,s): query in Value.find(dataset,v).value.lower(), choices) # THIS is very inefficient - rather do this # differently pager = Pager(choices, '.view', dataset=dataset.name, value=value.id, limit=10) # HACK: Fetch only the values on the selected page. value_objs = Value.id_map(dataset, map(lambda (c,v,s): v, pager.query[pager.offset:pager.offset+pager.limit])) for i, (c,v,s) in enumerate(pager.query): if v in value_objs: pager.query[i] = (c, value_objs.get(v), s) return render_template('value/view.html', dataset=dataset, value=value, values=pager, query=query)
def view(dataset, link): dataset = Dataset.find(dataset) link = Link.find(dataset, link) format = response_format() if format == 'json': return jsonify(link) return "Not implemented!"
def view(dataset, entity): dataset = Dataset.find(dataset) entity = Entity.find(dataset, entity) print entity.data format = response_format() if format == 'json': return jsonify(entity) query = request.args.get('query', '').strip().lower() choices = match_op(entity.name, dataset) choices = filter(lambda (c,e,s): e != entity.id, choices) if len(query): choices = filter(lambda (c,e,s): query in Entity.find(dataset,e).name.lower(), choices) # THIS is very inefficient - rather do this # differently pager = Pager(choices, '.view', dataset=dataset.name, entity=entity.id, limit=10) # HACK: Fetch only the entities on the selected page. entities = Entity.id_map(dataset, map(lambda (c,v,s): v, pager.query[pager.offset:pager.offset+pager.limit])) for i, (c,e,s) in enumerate(pager.query): if e in entities: pager.query[i] = (c, entities.get(e), s) return render_template('entity/view.html', dataset=dataset, entity=entity, entities=pager, query=query)
def match(dataset, alias, random=False): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) alias = Alias.find(dataset, alias) random = random or request.args.get('random')=='True' choices = match_op(alias.name, dataset, query=request.args.get('query')) pager = Pager(choices, '.match', dataset=dataset.name, alias=alias.id, limit=10) # HACK: Fetch only the entities on the selected page. entities = Entity.id_map(dataset, map(lambda (c,e,s): e, pager.query[pager.offset:pager.offset+pager.limit])) for i, (c,e,s) in enumerate(pager.query): if e in entities: pager.query[i] = (c, entities.get(e), s) html = render_template('alias/match.html', dataset=dataset, alias=alias, choices=pager, random=random) choice = 'INVALID' if alias.is_invalid else alias.entity_id if len(choices) and choice is None: c, e, s = choices[0] choice = 'INVALID' if s <= 50 else e.id return htmlfill.render(html, force_defaults=False, defaults={'choice': choice, 'name': alias.name, 'query': request.args.get('query', ''), 'random': random})
def edit(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_manage(dataset)) html = render_template('dataset/edit.html', dataset=dataset, algorithms=get_algorithms()) return htmlfill.render(html, defaults=dataset.as_dict())
def index(dataset, format='json'): dataset = Dataset.find(dataset) q = Alias.all(dataset, eager=True) if format == 'csv': fn = csv_filename(dataset, 'aliases') headers = {'Content-Disposition': 'attachment; filename=' + fn} return csvify(q, headers=headers) return jsonify(q)
def match(): dataset_arg = request.args.get('dataset') dataset = Dataset.find(dataset_arg) matches = find_matches(dataset, request.args.get('name'), filter=request.args.get('filter'), exclude=arg_int('exclude')) return query_pager(matches)
def match(): dataset_arg = request.args.get('dataset') dataset = Dataset.find(dataset_arg) matches = find_matches(dataset, request.args.get('name'), filter=request.args.get('filter'), exclude=arg_int('exclude')) pager = Pager(matches) return jsonify(pager.to_dict())
def upload(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) file_ = request.files.get('file') if not file_ or not file_.filename: err = {'file': "You need to upload a file"} raise Invalid("No file.", None, None, error_dict=err) upload = Upload.create(dataset, request.account, file_) db.session.commit() return jsonify(upload)
def review(dataset): entities = Entity.all() dataset = Dataset.find(dataset) entities = entities.filter_by(dataset=dataset) entities = entities.filter(Entity.reviewed == False) # noqa review_count = entities.count() if review_count == 0: return jsonify(None) entities = entities.offset(randint(0, review_count - 1)) return jsonify(entities.first())
def index(dataset, format='json'): dataset = Dataset.find(dataset) q = Entity.all(dataset, eager=True) if format == 'csv': fn = csv_filename(dataset, 'entities') headers = { 'Content-Disposition': 'attachment; filename=' + fn } return csvify(q, headers=headers) return jsonify(q)
def upload(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) file_ = request.files.get('file') if not file_ or not file_.filename: inv = Invalid("No file.", None, None, error_dict={'file': "You need to upload a file"}) raise inv upload = upload_file(dataset, file_, request.account) return redirect(url_for('.map', dataset=dataset.name, id=upload.id))
def review(dataset): entities = Entity.all() dataset = Dataset.find(dataset) entities = entities.filter_by(dataset=dataset) entities = entities.filter(Entity.reviewed == False) review_count = entities.count() if review_count == 0: return jsonify(None) entities = entities.offset(randint(0, review_count - 1)) return jsonify(entities.first())
def match_random(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) aliases = Alias.all_unmatched(dataset) count = aliases.count() if count == 0: return redirect(url_for('dataset.view', dataset=dataset.name)) alias = aliases.offset(randint(0, count - 1)).first() return redirect( url_for('.match', dataset=dataset.name, alias=alias.id, random=True))
def upload(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) file_ = request.files.get('file') if not file_ or not file_.filename: inv = Invalid("No file.", None, None, error_dict={'file': "You need to upload a file"}) return handle_invalid(inv, form, data={}, args=[dataset.name]) sig = upload_file(dataset, file_) return redirect(url_for('.map', dataset=dataset.name, sig=sig))
def get_authz(): permissions = {} dataset_name = request.args.get("dataset") if dataset_name is not None: dataset = Dataset.find(dataset_name) permissions[dataset_name] = { "view": True, "edit": authz.dataset_edit(dataset), "manage": authz.dataset_manage(dataset), } return jsonify(permissions)
def get_authz(): permissions = {} dataset_name = request.args.get('dataset') if dataset_name is not None: dataset = Dataset.find(dataset_name) permissions[dataset_name] = { 'view': True, 'edit': authz.dataset_edit(dataset), 'manage': authz.dataset_manage(dataset) } return jsonify(permissions)
def update(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_manage(dataset)) data = request_content() try: dataset.update(data) db.session.commit() flash("Updated %s" % dataset.label, "success") return redirect(url_for(".view", dataset=dataset.name)) except Invalid, inv: return handle_invalid(inv, edit, args=[dataset.name], data=data)
def match_random(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) aliases = Alias.all_unmatched(dataset) count = aliases.count() if count == 0: return redirect(url_for('dataset.view', dataset=dataset.name)) alias = aliases.offset(randint(0, count-1)).first() return redirect(url_for('.match', dataset=dataset.name, alias=alias.id, random=True))
def match_save(dataset, alias): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) alias = Alias.find(dataset, alias) random = request.form.get('random')=='True' data = request_content() try: alias.match(dataset, data, request.account) db.session.commit() except Invalid, inv: return handle_invalid(inv, match, data=data, args=[dataset.name, alias.id, random])
def update(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_manage(dataset)) data = request_content() try: flush_cache(dataset) dataset.update(data) db.session.commit() flash("Updated %s" % dataset.label, 'success') return redirect(url_for('.view', dataset=dataset.name)) except Invalid, inv: return handle_invalid(inv, edit, args=[dataset.name], data=data)
def submit(dataset, sig): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) data = request_content() entity_col = data.get('entity') or None alias_col = data.get('alias') or None if not (entity_col or alias_col): flash('You need to pick either a alias or entity column!', 'error') return map(dataset.name, sig) import_upload.delay(dataset.name, sig, request.account.id, entity_col, alias_col) flash('Loading data...', 'success') return redirect(url_for('dataset.view', dataset=dataset.name))
def submit(dataset, sig): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) data = request_content() value_col = data.get('value') or None link_col = data.get('link') or None if not (value_col or link_col): flash('You need to pick either a link or value column!', 'error') return map(dataset.name, sig) import_upload.delay(dataset.name, sig, request.account.id, value_col, link_col) flash('Loading data...', 'success') return redirect(url_for('dataset.view', dataset=dataset.name))
def submit(dataset, id): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) data = request_content() entity_col = data.get('entity') or None alias_col = data.get('alias') or None if not (entity_col or alias_col): flash('You need to pick either a alias or entity column!', 'error') return map(dataset.name, id) import_upload.delay(dataset.name, id, request.account.id, entity_col, alias_col) flash('Loading data...', 'success') return redirect(url_for('dataset.view', dataset=dataset.name))
def update(dataset, value): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) value = Value.find(dataset, value) data = request_content() try: value.update(data, request.account) db.session.commit() flash("Updated %s" % value.value, 'success') return redirect(url_for('.view', dataset=dataset.name, value=value.id)) except Invalid, inv: return handle_invalid(inv, view, data=data, args=[dataset.name, value.id])
def create(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) data = request_content() try: value = Value.create(dataset, data, request.account) db.session.commit() return redirect(url_for('.view', dataset=dataset.name, value=value.id)) except Invalid, inv: return handle_invalid(inv, view_dataset, data=data, args=[dataset.name])
def update(dataset, entity): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) entity = Entity.find(dataset, entity) data = request_content() try: entity.update(data, request.account) db.session.commit() flash("Updated %s" % entity.display_name, 'success') return redirect(url_for('.view', dataset=dataset.name, entity=entity.id)) except Invalid, inv: return handle_invalid(inv, view, data=data, args=[dataset.name, entity.id])
def create(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) data = request_content() try: entity = Entity.create(dataset, data, request.account) db.session.commit() return redirect( url_for('.view', dataset=dataset.name, entity=entity.id)) except Invalid, inv: return handle_invalid(inv, view_dataset, data=data, args=[dataset.name])
def match_save(dataset, link): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) link = Link.find(dataset, link) random = request.form.get('random')=='True' data = request_content() try: link.match(dataset, data, request.account) if link.value is not None: add_candidate_to_cache(dataset, link.key, link.value.id) db.session.commit() except Invalid, inv: return handle_invalid(inv, match, data=data, args=[dataset.name, link.id, random])
def match_save(dataset, alias): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) alias = Alias.find(dataset, alias) random = request.form.get('random') == 'True' data = request_content() try: alias.match(dataset, data, request.account) db.session.commit() except Invalid, inv: return handle_invalid(inv, match, data=data, args=[dataset.name, alias.id, random])
def merge(dataset, value): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) value = Value.find(dataset, value) data = request_content() try: target = value.merge_into(data, request.account) flush_cache(dataset) db.session.commit() flash("Merged %s" % value.value, 'success') return redirect(url_for('.view', dataset=dataset.name, value=target.id)) except Invalid, inv: print inv return handle_invalid(inv, view, data=data, args=[dataset.name, value.id])
def update(dataset, entity): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) entity = Entity.find(dataset, entity) data = request_content() try: entity.update(data, request.account) db.session.commit() flash("Updated %s" % entity.display_name, 'success') return redirect( url_for('.view', dataset=dataset.name, entity=entity.id)) except Invalid, inv: return handle_invalid(inv, view, data=data, args=[dataset.name, entity.id])
def view(dataset, value): dataset = Dataset.find(dataset) value = Value.find(dataset, value) format = response_format() if format == 'json': return jsonify(value) query = request.args.get('query', '').strip().lower() choices = match_op(value.value, dataset) choices = filter(lambda (c,v,s): v != value, choices) if len(query): choices = filter(lambda (c,v,s): query in v.value.lower(), choices) pager = Pager(choices, '.view', dataset=dataset.name, value=value.id, limit=10) return render_template('value/view.html', dataset=dataset, value=value, values=pager, query=query)
def view(dataset): dataset = Dataset.find(dataset) format = response_format() if format == 'json': return jsonify(dataset) unmatched = Alias.all_unmatched(dataset).count() entities = Entity.all(dataset, query=request.args.get('query')) pager = Pager(entities, '.view', dataset=dataset.name, limit=10) return render_template('dataset/view.html', entities=pager, num_entities=len(pager), num_aliases=Alias.all(dataset).count(), invalid=Alias.all_invalid(dataset).count(), query=request.args.get('query', ''), dataset=dataset, unmatched=unmatched)
def process(dataset, id): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) upload = Upload.find(dataset, id) mapping = request_data() mapping['reviewed'] = mapping.get('reviewed') or False mapping['columns'] = mapping.get('columns', {}) fields = mapping['columns'].values() for header in mapping['columns'].keys(): if header not in upload.tab.headers: raise Invalid("Invalid header: %s" % header, None, None) if 'name' not in fields and 'id' not in fields: raise Invalid("You have not selected a field that definies entity names.", None, None) import_upload.delay(upload.id, request.account.id, mapping) return jsonify({'status': 'Loading data...'})
def match(dataset, alias, random=False): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) alias = Alias.find(dataset, alias) random = random or request.args.get('random') == 'True' choices = match_op(alias.name, dataset, query=request.args.get('query')) pager = Pager(choices, '.match', dataset=dataset.name, alias=alias.id, limit=10) # HACK: Fetch only the entities on the selected page. entities = Entity.id_map( dataset, map(lambda (c, e, s): e, pager.query[pager.offset:pager.offset + pager.limit])) for i, (c, e, s) in enumerate(pager.query): if e in entities: pager.query[i] = (c, entities.get(e), s) html = render_template('alias/match.html', dataset=dataset, alias=alias, choices=pager, random=random) choice = 'INVALID' if alias.is_invalid else alias.entity_id if len(choices) and choice is None: c, e, s = choices[0] choice = 'INVALID' if s <= 50 else e.id return htmlfill.render(html, force_defaults=False, defaults={ 'choice': choice, 'name': alias.name, 'query': request.args.get('query', ''), 'random': random })
def index(): entities = Entity.all() dataset_arg = request.args.get('dataset') if dataset_arg is not None: dataset = Dataset.find(dataset_arg) entities = entities.filter_by(dataset=dataset) filter_name = request.args.get('filter_name', '') if len(filter_name): query = '%' + filter_name + '%' entities = entities.filter(Entity.name.ilike(query)) # TODO, other filters. format = request.args.get('format', 'json').lower().strip() if format == 'csv': res = csvify(entities) else: res = query_pager(entities) if arg_bool('download'): fn = dataset_filename(dataset, format) res.headers['Content-Disposition'] = 'attachment; filename=' + fn return res
def import_upload(dataset_name, sig, account_id, entity_col, alias_col): dataset = Dataset.find(dataset_name) account = Account.by_id(account_id) metadata, row_set = parse_upload(dataset, sig) headers = detect_headers(row_set) for row in row_set: data = dict([(c.column, c.value) for c in row]) entity = data.pop(entity_col) if entity_col else None alias = data.pop(alias_col) if alias_col else None if alias_col and alias is not None and len(alias) and alias != entity: d = {'name': alias, 'data': data} alias_obj = Alias.lookup(dataset, d, account, match_entity=False) data = {} if entity_col and entity is not None and len(entity): d = {'name': entity, 'data': data} entity_obj = Entity.by_name(dataset, entity) if entity_obj is None: entity_obj = Entity.create(dataset, d, account) entity_obj.data = data if alias_col and entity_col: alias_obj.match(dataset, {'choice': entity_obj.id}, account) db.session.commit() flush_cache()
def lookup(dataset): dataset = Dataset.find(dataset) readonly = validators.StringBool(if_empty=False, if_missing=False)\ .to_python(request.args.get('readonly')) readonly = readonly if authz.logged_in() else True data = request_content() if response_format() != 'json': return Response("Not implemented!", status=400) try: alias = Alias.lookup(dataset, data, request.account, readonly=readonly) if alias is None: return jsonify( { 'is_matched': False, 'entity': None, 'name': data.get('name'), 'dataset': dataset.name }, status=404) if isinstance(alias, Entity): return jsonify( { 'is_matched': True, 'entity': alias, 'name': data.get('name'), 'dataset': dataset.name }, status=200) db.session.commit() status = 200 if alias.is_matched else 404 status = 418 if alias.is_invalid else status return jsonify(alias, status=status) except Invalid, inv: return handle_invalid(inv, index, data=data, args=[dataset.name])
def view(dataset, entity): dataset = Dataset.find(dataset) entity = Entity.find(dataset, entity) print entity.data format = response_format() if format == 'json': return jsonify(entity) query = request.args.get('query', '').strip().lower() choices = match_op(entity.name, dataset) choices = filter(lambda (c, e, s): e != entity.id, choices) if len(query): choices = filter( lambda (c, e, s): query in Entity.find(dataset, e).name.lower(), choices) # THIS is very inefficient - rather do this # differently pager = Pager(choices, '.view', dataset=dataset.name, entity=entity.id, limit=10) # HACK: Fetch only the entities on the selected page. entities = Entity.id_map( dataset, map(lambda (c, v, s): v, pager.query[pager.offset:pager.offset + pager.limit])) for i, (c, e, s) in enumerate(pager.query): if e in entities: pager.query[i] = (c, entities.get(e), s) return render_template('entity/view.html', dataset=dataset, entity=entity, entities=pager, query=query)
def by_name(dataset): dataset = Dataset.find(dataset) name = request.args.get('name') entity = object_or_404(Entity.by_name(dataset, name)) return jsonify(entity)
def view(dataset, id): dataset = Dataset.find(dataset) authz.require(authz.dataset_edit(dataset)) upload = Upload.find(dataset, id) return jsonify(upload)
def view(dataset): dataset = Dataset.find(dataset) return jsonify(dataset)
def attributes(dataset): dataset = Dataset.find(dataset) return jsonify({'attributes': attribute_keys(dataset)})
def update(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_manage(dataset)) dataset.update(request_data()) db.session.commit() return redirect(url_for('.view', dataset=dataset.name))
def view_by_name(dataset): dataset = Dataset.find(dataset) alias = Alias.by_name(dataset, request.args.get('name')) if alias is None: raise NotFound("No such alias: %s" % request.args.get('name')) return view(dataset.name, alias.id)
def view(dataset, alias): dataset = Dataset.find(dataset) alias = Alias.find(dataset, alias) #format = response_format() #if format == 'json': return jsonify(alias)
def view_by_name(dataset): dataset = Dataset.find(dataset) entity = Entity.by_name(dataset, request.args.get('name')) if entity is None: raise NotFound("No such entity: %s" % request.args.get('name')) return view(dataset.name, entity.id)
def delete(dataset): dataset = Dataset.find(dataset) authz.require(authz.dataset_manage(dataset)) dataset.delete() db.session.commit() return ('', 204)