def get_results(query, limit): sources = {} for i, row in enumerate(scan_iter(query)): if i >= limit: return data = { 'file_url': url_for('documents_api.file', document_id=row.get('_id')) } for name, value in row.get('_source').items(): if name == 'source_id': if value not in sources: source = Source.by_id(value) if source is None: sources[value] = '[Deleted source %s]' % value else: sources[value] = source.label value = sources[value] name = 'source' if name not in FIELDS: continue if isinstance(value, (list, tuple, set)): value = ', '.join(value) data[name] = value yield data
def alert_query(alert): """Execute the query and return a set of results.""" q = text_query(alert.query_text) q = authz_sources_filter(q) if alert.entity_id: q = filter_query(q, [('entities.uuid', alert.entity_id)], OR_FIELDS) if alert.notified_at: q = add_filter(q, {"range": {"created_at": {"gt": alert.notified_at}}}) q = {'query': q, 'size': 150} result, hits, output = execute_basic(TYPE_DOCUMENT, q) sub_queries = [] sources = {} for doc in hits.get('hits', []): document = doc.get('_source') document['id'] = int(doc.get('_id')) source_id = document['source_id'] if source_id not in sources: sources[source_id] = Source.by_id(source_id) if sources[source_id] is None: continue document['source'] = sources[source_id] document['records'] = {'results': [], 'total': 0} sq = records_query(document['id'], alert.to_query(), size=1) if sq is not None: sub_queries.append(json.dumps({})) sub_queries.append(json.dumps(sq)) output['results'].append(document) run_sub_queries(output, sub_queries) return output
def update(id): authz.require(authz.source_write(id)) source = obj_or_404(Source.by_id(id)) source.update(request_data()) db.session.add(source) db.session.commit() return view(id)
def format_results(query): sources = {} entities = {} results = [] for row in raw_iter(query): src = row.get('_source') data = {} for name, value in src.items(): if isinstance(value, dict) or name in SKIP_FIELDS: continue if name == 'entities': load_ids = [] for entity_id in value: if entity_id not in entities: load_ids.append(entity_id) if len(load_ids): for id, ent in Entity.by_id_set(load_ids).items(): entities[id] = ent.name value = ', '.join([entities.get(e) for e in value if entities.get(e) is not None]) if isinstance(value, (list, tuple, set)): value = ', '.join(value) if name == 'source_id': # WARNING: don't to one query per row if value not in sources: source = Source.by_id(value) if source is None: sources[value] = '[Deleted source %s]' % value else: sources[value] = source.label value = sources[value] data[name] = value results.append(data) return results
def test_delete_source(self): source = Source.by_id(1000) res = self.client.get('/api/1/query?q="mention fruit"') assert res.json['total'] == 1, res.json delete_source(source.id) optimize_search() res = self.client.get('/api/1/query?q="mention fruit"') assert res.json['total'] == 0, res.json
def alert_query(alert): """Execute the query and return a set of results.""" q = text_query(alert.query_text) q = authz_sources_filter(q) if alert.entity_id: q = filter_query(q, [('entities.uuid', alert.entity_id)], OR_FIELDS) if alert.notified_at: q = add_filter(q, { "range": { "created_at": { "gt": alert.notified_at } } }) q = { 'query': q, 'size': 150 } result, hits, output = execute_basic(TYPE_DOCUMENT, q) sub_queries = [] sources = {} for doc in hits.get('hits', []): document = doc.get('_source') document['id'] = int(doc.get('_id')) source_id = document['source_id'] if source_id not in sources: sources[source_id] = Source.by_id(source_id) if sources[source_id] is None: continue document['source'] = sources[source_id] document['records'] = {'results': [], 'total': 0} sq = records_query(document['id'], alert.to_query(), size=1) if sq is not None: sub_queries.append(json.dumps({})) sub_queries.append(json.dumps(sq)) output['results'].append(document) run_sub_queries(output, sub_queries) return output
def process(id): authz.require(authz.source_write(id)) source = obj_or_404(Source.by_id(id)) analyze_source.delay(source.id) return jsonify({'status': 'ok'})
def view(id): authz.require(authz.source_read(id)) source = obj_or_404(Source.by_id(id)) return jsonify(source)
def view(id): authz.require(authz.source_read(id)) source = obj_or_404(Source.by_id(id)) etag_cache_keygen(source) return jsonify(source)
def get_source(sid): if sid not in SRCS: src = Source.by_id(sid) SRCS[sid] = src and src.label or '' return SRCS[sid]