def index():
authz.require(authz.system_read())
q = Context.all()
if arg_bool('imports'):
q = q.filter(Context.resource_name != None) # noqa
q = q.order_by(Context.updated_at.desc())
return jsonify(Pager(q))
def update(id):
authz.require(authz.system_edit())
entity = obj_or_404(EntityQuery.by_id(id))
context = Context.create(current_user, {})
entity.update(request_data(), context)
db.session.commit()
return redirect(url_for(".view", id=entity.id))
def load_next(root):
authz.require(authz.system_edit())
q = Context.by_root(root)
q = q.filter(Context.enrich_status == PENDING)
q = q.order_by(Context.enrich_score.desc())
context = q.first()
if context is None:
# TODO: spider status system
return jsonify({"status": "wait"})
return jsonify({"status": "next", "next": context.id})
def store_upload(file, filename, user):
ctx = {'active': False}
context = Context.create(user, ctx)
db.session.add(context)
db.session.flush()
package = get_package(context)
meta = {'source_file': filename}
source = package.ingest(file, meta=meta, overwrite=False)
context.resource_name = source.path
context.source_url = source.url
db.session.flush()
return context
def store(root):
authz.require(authz.system_edit())
ctx_data = request_data()
q = Context.by_root(root)
q = q.filter(Context.id == ctx_data.get("id"))
context = q.first()
if context is None:
return jsonify({"status": "failed"}, status=400)
context.update(request_data())
context.active = context.enrich_status == ACCEPTED
context.user = current_user
db.session.commit()
# TODO: task newly activated entities.
return jsonify({"status": "ok"})
def view(root, id):
authz.require(authz.system_edit())
q = Context.by_root(root)
q = q.filter(Context.id == id)
context = obj_or_404(q.first())
statements = []
entities = set()
for stmt in context.statements:
entities.add(stmt.subject)
if qualified[stmt.attribute].data_type == "entity":
entities.add(stmt._value)
statements.append(stmt.to_dict(raw=True))
q = [{"assume": [context.id], "id|=": list(entities), "label": None, "type": None}]
entities = {e.get("id"): e for e in execute_query(q).get("result")}
return jsonify(
{"status": "ok", "entities": entities, "context": context.to_dict(enrich=True), "statements": statements}
)
def load_upload(context_id):
context = Context.by_id(context_id)
if not context.resource_name or not context.resource_mapping:
log.warning("No resource associated with context: %r", context)
return
context.active = False
db.session.commit()
source, table = get_table(context)
set_state(source, 'loading')
handler = get_logger(source.package, context)
try:
for record in table.records():
try:
load_entity(context, context.resource_mapping, record)
except DataException, de:
log.error("Cannot convert '%s' to %s for attribute '%s': %s",
de.value, de.data_type, de.attribute, de.message)
except Exception, e:
log.exception(e)
def analyze_upload(context_id):
context = Context.by_id(context_id)
if not context.resource_name:
log.warning("No resource associated with context: %r", context)
return
source, target = get_table(context)
set_state(source, 'analyzing')
handler = get_logger(source.package, context)
try:
log.info("Extracting tabular data...")
if not source.exists():
log.warning("Source data does not exist: %r",
context.resource_name)
return
operator = TableExtractOperator(None, 'temp', {})
operator.transform(source, target)
set_state(source, 'analyzed')
except Exception, e:
log.error(unicode(e))
set_state(source, 'failed')
def create_context(self, root=None, url=None, score=None):
q = db.session.query(Context)
q = q.filter(Context.enrich_root == root)
q = q.filter(Context.source_url == url)
ctx = q.first()
if ctx is not None:
if ctx.enrich_score == score or ctx.enrich_status == ACCEPTED:
raise ContextExistsException()
if ctx.enrich_status == REJECTED:
ctx.enrich_status = PENDING
else:
ctx = Context.create(None, {
'active': False,
'source_url': url,
'enrich_root': root,
'enrich_score': score,
'enrich_status': PENDING,
'publisher': self.PUBLISHER_LABEL,
'publisher_url': self.PUBLISHER_URL
})
ctx.score = score
db.session.add(ctx)
return ctx
def update(id):
authz.require(authz.system_edit())
context = obj_or_404(Context.by_id(id))
context.update(request_data())
db.session.commit()
return redirect(url_for('.view', id=context.id))
def view(id):
authz.require(authz.system_read())
context = obj_or_404(Context.by_id(id))
return jsonify(context)
def create():
authz.require(authz.system_edit())
context = Context.create(current_user, request_data())
db.session.commit()
return redirect(url_for('.view', id=context.id))
def create():
authz.require(authz.system_edit())
context = Context.create(current_user, {})
entity = Entity.create(request_data(), context)
db.session.commit()
return redirect(url_for(".view", id=entity.id))
