def index(): authz.require(authz.system_read()) q = Context.all() if arg_bool('imports'): q = q.filter(Context.resource_name != None) # noqa q = q.order_by(Context.updated_at.desc()) return jsonify(Pager(q))
def update(id): authz.require(authz.system_edit()) entity = obj_or_404(EntityQuery.by_id(id)) context = Context.create(current_user, {}) entity.update(request_data(), context) db.session.commit() return redirect(url_for(".view", id=entity.id))
def load_next(root): authz.require(authz.system_edit()) q = Context.by_root(root) q = q.filter(Context.enrich_status == PENDING) q = q.order_by(Context.enrich_score.desc()) context = q.first() if context is None: # TODO: spider status system return jsonify({"status": "wait"}) return jsonify({"status": "next", "next": context.id})
def store_upload(file, filename, user): ctx = {'active': False} context = Context.create(user, ctx) db.session.add(context) db.session.flush() package = get_package(context) meta = {'source_file': filename} source = package.ingest(file, meta=meta, overwrite=False) context.resource_name = source.path context.source_url = source.url db.session.flush() return context
def store(root): authz.require(authz.system_edit()) ctx_data = request_data() q = Context.by_root(root) q = q.filter(Context.id == ctx_data.get("id")) context = q.first() if context is None: return jsonify({"status": "failed"}, status=400) context.update(request_data()) context.active = context.enrich_status == ACCEPTED context.user = current_user db.session.commit() # TODO: task newly activated entities. return jsonify({"status": "ok"})
def view(root, id): authz.require(authz.system_edit()) q = Context.by_root(root) q = q.filter(Context.id == id) context = obj_or_404(q.first()) statements = [] entities = set() for stmt in context.statements: entities.add(stmt.subject) if qualified[stmt.attribute].data_type == "entity": entities.add(stmt._value) statements.append(stmt.to_dict(raw=True)) q = [{"assume": [context.id], "id|=": list(entities), "label": None, "type": None}] entities = {e.get("id"): e for e in execute_query(q).get("result")} return jsonify( {"status": "ok", "entities": entities, "context": context.to_dict(enrich=True), "statements": statements} )
def load_upload(context_id): context = Context.by_id(context_id) if not context.resource_name or not context.resource_mapping: log.warning("No resource associated with context: %r", context) return context.active = False db.session.commit() source, table = get_table(context) set_state(source, 'loading') handler = get_logger(source.package, context) try: for record in table.records(): try: load_entity(context, context.resource_mapping, record) except DataException, de: log.error("Cannot convert '%s' to %s for attribute '%s': %s", de.value, de.data_type, de.attribute, de.message) except Exception, e: log.exception(e)
def analyze_upload(context_id): context = Context.by_id(context_id) if not context.resource_name: log.warning("No resource associated with context: %r", context) return source, target = get_table(context) set_state(source, 'analyzing') handler = get_logger(source.package, context) try: log.info("Extracting tabular data...") if not source.exists(): log.warning("Source data does not exist: %r", context.resource_name) return operator = TableExtractOperator(None, 'temp', {}) operator.transform(source, target) set_state(source, 'analyzed') except Exception, e: log.error(unicode(e)) set_state(source, 'failed')
def create_context(self, root=None, url=None, score=None): q = db.session.query(Context) q = q.filter(Context.enrich_root == root) q = q.filter(Context.source_url == url) ctx = q.first() if ctx is not None: if ctx.enrich_score == score or ctx.enrich_status == ACCEPTED: raise ContextExistsException() if ctx.enrich_status == REJECTED: ctx.enrich_status = PENDING else: ctx = Context.create(None, { 'active': False, 'source_url': url, 'enrich_root': root, 'enrich_score': score, 'enrich_status': PENDING, 'publisher': self.PUBLISHER_LABEL, 'publisher_url': self.PUBLISHER_URL }) ctx.score = score db.session.add(ctx) return ctx
def update(id): authz.require(authz.system_edit()) context = obj_or_404(Context.by_id(id)) context.update(request_data()) db.session.commit() return redirect(url_for('.view', id=context.id))
def view(id): authz.require(authz.system_read()) context = obj_or_404(Context.by_id(id)) return jsonify(context)
def create(): authz.require(authz.system_edit()) context = Context.create(current_user, request_data()) db.session.commit() return redirect(url_for('.view', id=context.id))
def create(): authz.require(authz.system_edit()) context = Context.create(current_user, {}) entity = Entity.create(request_data(), context) db.session.commit() return redirect(url_for(".view", id=entity.id))