def update_entity_full(entity_id): """Perform some update operations on entities.""" query = db.session.query(Entity).filter(Entity.id == entity_id) entity = query.first() generate_entity_references(entity) reindex_entity(entity) Alert.dedupe(entity.id)
def test_db_alert_dates(): al = Alert(query = 'test data', user = get_test_user(), label = 'test query', checking_interval = 5) db.session.add(al) db.session.commit() assert al.due_to_check() al.mark_as_checked() db.session.add(al) db.session.commit() assert not al.due_to_check() al.checked_at = datetime.datetime.utcnow() - datetime.timedelta(days=6) db.session.add(al) db.session.commit() assert al.due_to_check() al.checked_at = datetime.datetime.utcnow() - datetime.timedelta(days=4) db.session.add(al) db.session.commit() assert not al.due_to_check() db.session.delete(al) db.session.commit()
def test_notify(self): data = {'query': 'Kashmir'} failed_alert = Alert.create(data, self.role_no_email.id) failed_alert.notified_at = datetime.utcnow() - timedelta(hours=72) alert = Alert.create(data, self.role_email.id) alert.notified_at = datetime.utcnow() + timedelta(hours=72) db.session.commit() res = get_notifications(self.role_email) notcount = res.get('hits').get('total').get('value') assert notcount == 0, notcount db.session.refresh(alert) alert.notified_at = datetime.utcnow() - timedelta(hours=72) db.session.add(alert) db.session.commit() check_alerts() res = get_notifications(self.role_email) notcount = res.get('hits').get('total').get('value') assert notcount == 1, res.get('hits') check_alerts() res = get_notifications(self.role_email) notcount = res.get('hits').get('total').get('value') assert notcount == 1, res.get('hits')
def create(): # also handles update data = request.get_json() print(data) if 'query_text' not in data: return jsonify({'status': 'invalid'}) authz.require(authz.logged_in()) if data.get('alert_id', None): # UPDATE alert_id = int(data['alert_id']) alert = obj_or_404(Alert.by_id(alert_id)) authz.require(alert.role_id == request.auth_role.id) alert.query_text = data['query_text'] alert.custom_label = data.get('custom_label' '') or data['query_text'] alert.checking_interval=int(data.get('checking_interval', 9)) else: # CREATE alert = Alert( role_id = request.auth_role.id, query_text=data['query_text'], custom_label=data.get('custom_label' '') or data['query_text'], checking_interval=int(data.get('checking_interval', 9)) ) db.session.add(alert) db.session.commit() return view(alert.id)
def update_entity_full(entity_id): """Perform update operations on entities.""" query = db.session.query(Entity).filter(Entity.id == entity_id) entity = query.first() if entity is None: log.error("No entity with ID: %r", entity_id) return Alert.dedupe(entity.id) index_entity(entity)
def cleanup_deleted(): from aleph.model import Alert, Entity, Collection from aleph.model import Permission, Role Alert.cleanup_deleted() Permission.cleanup_deleted() Entity.cleanup_deleted() Collection.cleanup_deleted() Role.cleanup_deleted() db.session.commit()
def update_entity_full(entity_id): """Perform update operations on entities.""" query = db.session.query(Entity).filter(Entity.id == entity_id) entity = query.first() generate_entity_references(entity) with graph.transaction() as tx: graph.load_entity(tx, entity) graph.generate_paths(tx, entity) reindex_entity(entity) Alert.dedupe(entity.id)
def cleanup_deleted(): from aleph.model import Alert, Entity, Collection from aleph.model import Permission, Role, Document from aleph.model import Diagram, Mapping Mapping.cleanup_deleted() Diagram.cleanup_deleted() Document.cleanup_deleted() Alert.cleanup_deleted() Permission.cleanup_deleted() Entity.cleanup_deleted() Collection.cleanup_deleted() Role.cleanup_deleted() db.session.commit()
def delete(id): authz.require(authz.logged_in()) alert = obj_or_404(Alert.by_id(id, role=request.auth_role)) alert.delete() db.session.commit() log_event(request) return jsonify({'status': 'ok'})
def view(alert_id): """Return the alert with id `alert_id`. --- get: summary: Fetch an alert parameters: - description: The alert ID. in: path name: alert_id required: true schema: minimum: 1 type: integer example: 2 responses: '200': content: application/json: schema: $ref: '#/components/schemas/Alert' description: OK tags: - Alert """ require(request.authz.logged_in) alert = obj_or_404(Alert.by_id(alert_id, role_id=request.authz.id)) return AlertSerializer.jsonify(alert)
def create(): authz.require(authz.logged_in()) alert = Alert.create(request_data(), request.auth_role) db.session.commit() log_event(request) return view(alert.id)
def delete(alert_id): """Delete the alert with id `alert_id`. --- delete: summary: Delete an alert parameters: - description: The alert ID. in: path name: alert_id required: true schema: minimum: 1 type: integer example: 2 responses: '204': description: No Content tags: - Alert """ require(request.authz.session_write) alert = obj_or_404(Alert.by_id(alert_id, role_id=request.authz.id)) alert.delete() db.session.commit() return ("", 204)
def test_notify(self): data = {'query_text': '', 'label': 'Test Alert'} alert = Alert.create(data, self.role_email) alert.notified_at = datetime.utcnow() + timedelta(hours=72) db.session.commit() with mail.record_messages() as outbox: check_alerts() assert len(outbox) == 0, outbox db.session.refresh(alert) alert.notified_at = datetime.utcnow() - timedelta(hours=72) db.session.add(alert) db.session.commit() with mail.record_messages() as outbox: check_alerts() assert len(outbox) == 1, outbox msg = outbox[0] assert 'Test Alert' in msg.subject, msg assert '*****@*****.**' in msg.recipients, msg with mail.record_messages() as outbox: check_alerts() assert len(outbox) == 0, outbox
def check_role_alerts(authz): alerts = Alert.by_role(authz.role).all() if not len(alerts): return log.info('Alerting %r, %d alerts...', authz.role, len(alerts)) for alert in alerts: args = {'q': alert.query_text} entity = get_entity(alert.entity_id) if alert.entity_id else None state = SearchQueryParser(args, authz) query = AlertDocumentsQuery(state, entity=entity, since=alert.notified_at) results = query.search().get('hits') if results['total'] == 0: continue log.info('Found %d new results for: %r', results['total'], alert.label) alert.update() try: subject = '%s (%s new results)' % (alert.label, results['total']) html = render_template('email/alert.html', alert=alert, role=authz.role, total=results.get('total'), results=format_results(alert, results), app_title=app_title, ui_url=app_ui_url) notify_role(authz.role, subject, html) except Exception as ex: log.exception(ex) db.session.commit()
def delete(id): request.authz.require(request.authz.session_write()) alert = obj_or_404(Alert.by_id(id, role=request.authz.role)) alert.delete() db.session.commit() log_event(request) return jsonify({'status': 'ok'})
def delete(id): authz.require(authz.logged_in()) alert = obj_or_404(Alert.by_id(id)) authz.require(alert.user_id == current_user.id) db.session.delete(alert) db.session.commit() return jsonify({'status': 'ok'})
def check_role_alerts(authz): alerts = Alert.by_role(authz.role).all() if not len(alerts): return log.info('Alerting %r, %d alerts...', authz.role, len(alerts)) for alert in alerts: args = { 'q': alert.query_text, 'filter:entities.id': alert.entity_id, 'limit': 50 } state = QueryState(args, authz) results = documents_query(state, since=alert.notified_at) if results['total'] == 0: continue log.info('Found %d new results for: %r', results['total'], alert.label) alert.update() try: subject = '%s (%s new results)' % (alert.label, results['total']) html = render_template('email/alert.html', alert=alert, role=authz.role, total=results.get('total'), results=format_results(alert, results), app_title=app_title, app_url=app_url) notify_role(authz.role, subject, html) except Exception as ex: log.exception(ex) db.session.commit()
def check_role_alerts(role): alerts = Alert.by_role(role).all() if not len(alerts): return log.info('Alerting %r, %d alerts...', role, len(alerts)) for alert in alerts: results = alert_query(alert) if results['total'] == 0: continue log.info('Found: %d new results for: %r', results['total'], alert.label) alert.update() try: subject = '%s (%s new results)' % (alert.label, results['total']) html = render_template('alert.html', alert=alert, results=results, role=role, qs=make_document_query(alert), app_title=get_config('APP_TITLE'), app_url=get_app_url()) notify_role(role, subject, html) except Exception as ex: log.exception(ex) db.session.commit()
def check_alert(alert_id): alert = Alert.by_id(alert_id) if alert is None or alert.role is None: return if not alert.role.is_alertable: return authz = Authz.from_role(alert.role) query = alert_query(alert, authz) index = entities_read_index(schema=Entity.THING) result = es.search(index=index, body=query) for result in result.get('hits').get('hits', []): entity = unpack_result(result) if entity is None: continue log.info('Alert [%s]: %s', alert.query, entity.get('name')) params = { 'alert': alert, 'role': alert.role, 'entity': entity } publish(Events.MATCH_ALERT, actor_id=entity.get('uploader_id'), params=params) alert.update() db.session.commit() db.session.close()
def check_alert(alert_id): alert = Alert.by_id(alert_id) if alert is None or alert.role is None: return log.info("Check alert [%s]: %s", alert.id, alert.query) authz = Authz.from_role(alert.role) query = alert_query(alert, authz) index = entities_read_index(schema=Entity.THING) result = es.search(index=index, body=query) for result in result.get('hits').get('hits', []): entity = unpack_result(result) if entity is None: continue log.info('Alert [%s]: %s', alert.query, entity.get('name')) params = { 'alert': alert, 'role': alert.role, 'entity': entity.get('id') } publish(Events.MATCH_ALERT, params=params, channels=[alert.role]) db.session.flush() alert.update() db.session.commit() db.session.close()
def create(): require(request.authz.session_write) data = parse_request(AlertSchema) alert = Alert.create(data, request.authz.id) db.session.commit() tag_request(alert_id=alert.id) return AlertSerializer.jsonify(alert)
def test_delete(self): data = {'query': {'q': 'putin'}} jdata = json.dumps(data) self.login() res = self.client.post('/api/1/alerts', data=jdata, content_type='application/json') assert res.status_code == 200, res.json count = Alert.all().count() url = '/api/1/alerts/%s' % res.json['id'] res = self.client.delete(url) assert res.status_code == 200, res.json new_count = Alert.all().count() real_count = db.session.query(Alert).count() assert count == real_count, (count, real_count) assert new_count == real_count - 1
def create(): """Creates an alert for a given query string. --- post: summary: Create an alert requestBody: content: application/json: schema: $ref: '#/components/schemas/AlertCreate' responses: '200': content: application/json: schema: $ref: '#/components/schemas/Alert' description: OK tags: - Alert """ require(request.authz.session_write) data = parse_request("AlertCreate") alert = Alert.create(data, request.authz.id) db.session.commit() tag_request(alert_id=alert.id) return AlertSerializer.jsonify(alert)
def index(): """Returns a list of alerts for the user. --- get: summary: List alerts responses: '200': content: application/json: schema: type: object allOf: - $ref: '#/components/schemas/QueryResponse' properties: results: type: array items: $ref: '#/components/schemas/Alert' description: OK tags: - Alert """ require(request.authz.logged_in) query = Alert.by_role_id(request.authz.id) result = DatabaseQueryResult(request, query) return AlertSerializer.jsonify_result(result)
def check_alert(alert_id): alert = Alert.by_id(alert_id) if alert is None or alert.role is None: return log.info("Check alert [%s]: %s", alert.id, alert.query) authz = Authz.from_role(alert.role) try: query = alert_query(alert, authz) index = entities_read_index(schema=Entity.THING) result = es.search(index=index, body=query) except RequestError as re: log.error("Invalid query [%s]: %r", alert.query, re.error) alert.delete() db.session.commit() return for result in result.get("hits").get("hits", []): entity = unpack_result(result) if entity is None: continue log.info("Alert [%s]: %s", alert.query, entity.get("id")) params = { "alert": alert, "role": alert.role, "entity": entity.get("id"), "collection": entity.get("collection_id"), } channels = [alert.role] # channels.append(channel_tag(collection_id, Collection)) publish(Events.MATCH_ALERT, params=params, channels=channels) alert.update() db.session.commit()
def test_notify_no_email(self): data = {'query': {}, 'custom_label': 'Test Alert'} assert self.role_no_email.email is None, self.role_no_email.email alert = Alert.create(data, self.role_no_email) alert.notified_at = datetime.utcnow() - timedelta(hours=72) db.session.commit() notcount = Notification.all().count() assert notcount == 0, notcount
def _resolve_alerts(self, cache): alerts = set() for (type_, id_) in cache.keys(): if type_ == Alert: alerts.add(id_) if not len(alerts): return for alert in Alert.all_by_ids(alerts, deleted=True): cache[(Alert, str(alert.id))] = role
def setUp(self): super(EntitiesTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Original Collection' self.col.foreign_id = 'test_coll_entities' db.session.add(self.col) self.col_other = Collection() self.col_other.label = 'Other Collection' self.col_other.foreign_id = 'test_coll_entities_other' db.session.add(self.col_other) db.session.flush() self.ent = Entity.save( { 'name': 'Winnie the Pooh', 'jurisdiction_code': 'pa', 'summary': 'a fictional teddy bear created by author A. A. Milne', 'identifiers': [{ 'scheme': 'wikipedia', 'identifier': 'en:Winnie-the-Pooh' }], 'other_names': [{ 'name': u'Puh der Bär' }, { 'name': 'Pooh Bear' }] }, [self.col]) db.session.add(self.ent) db.session.flush() self.other = Entity.save( { 'name': 'Pu der Bär', 'jurisdiction_code': 'de', 'description': 'he is a bear', 'identifiers': [{ 'scheme': 'wikipedia', 'identifier': 'en:Winnie-the-Pooh' }, { 'scheme': 'animals', 'identifier': 'bears.winnie.pooh' }], 'other_names': [{ 'name': u'Puh der Bär' }] }, [self.col_other]) db.session.add(self.other) self.alert = Alert() self.alert.entity = self.other db.session.add(self.alert) db.session.commit()
def test_notify_no_email(self): data = {'query': {}, 'custom_label': 'Test Alert'} assert self.role_no_email.email is None, self.role_no_email.email alert = Alert.create(data, self.role_no_email) alert.notified_at = datetime.utcnow() - timedelta(hours=72) db.session.commit() with mail.record_messages() as outbox: check_alerts() assert len(outbox) == 0, outbox
def test_notify_entity(self): data = {'query': 'kwazulu'} alert = Alert.create(data, self.role_email.id) alert.notified_at = datetime.utcnow() - timedelta(hours=72) db.session.add(alert) db.session.commit() check_alerts() notcount = Notification.all().count() assert notcount == 2, notcount
def test_notify_entity(self): data = {'query_text': 'kwazulu', 'label': 'Test Alert'} alert = Alert.create(data, self.role_email.id) alert.notified_at = datetime.utcnow() - timedelta(hours=72) db.session.add(alert) db.session.commit() check_alerts() notcount = Notification.all().count() assert notcount == 2, notcount
def create(): authz.require(authz.logged_in()) data = request.get_json() validate(data, alerts_schema) alert = Alert(user_id=current_user.id, query=data['query'], label=data.get('custom_label', data['query']), checking_interval=int(data.get('checking_interval', 9))) db.session.add(alert) db.session.commit() return view(alert.id)
def test_delete(self): data = {"query": "putin"} jdata = json.dumps(data) _, headers = self.login() res = self.client.post( "/api/2/alerts", data=jdata, headers=headers, content_type="application/json", ) assert res.status_code == 200, res.status_code count = Alert.all().count() url = "/api/2/alerts/%s" % res.json["id"] res = self.client.delete(url, headers=headers) assert res.status_code == 204, res.json new_count = Alert.all().count() real_count = db.session.query(Alert).count() assert new_count == real_count, (count, real_count) assert new_count == (count - 1), (count, new_count)
def query(): creds = authz.collections(authz.READ), authz.sources(authz.READ) enable_cache(vary_user=True, vary=creds) query = documents_query(request.args) query['size'] = get_limit(default=100) query['from'] = get_offset() result = execute_documents_query(request.args, query) result['alert'] = None if authz.logged_in(): result['alert'] = Alert.exists(request.args, request.auth_role) params = next_params(request.args, result) if params is not None: result['next'] = url_for('search_api.query', **params) return jsonify(result)
def test_notify(self): data = {'query': 'fruit'} alert = Alert.create(data, self.role_email.id) alert.notified_at = datetime.utcnow() + timedelta(hours=72) db.session.commit() notcount = Notification.all().count() assert notcount == 0, notcount db.session.refresh(alert) alert.notified_at = datetime.utcnow() - timedelta(hours=72) db.session.add(alert) db.session.commit() check_alerts() notcount = Notification.all().count() assert notcount == 1, notcount check_alerts() notcount = Notification.all().count() assert notcount == 1, notcount
def check_role_alerts(role): alerts = Alert.by_role(role).all() if not len(alerts): return log.info('Alerting %r, %d alerts...', role, len(alerts)) for alert in alerts: results = alert_query(alert) if results['total'] == 0: continue log.info('Found: %d new results for: %r', results['total'], alert.label) alert.update() try: subject = '%s (%s new results)' % (alert.label, results['total']) html = render_template('alert.html', alert=alert, results=results, role=role, qs=make_document_query(alert), app_title=get_app_title(), app_url=get_app_url()) notify_role(role, subject, html) except Exception as ex: log.exception(ex) db.session.commit()
def create(): ''' should also handle update''' authz.require(authz.logged_in()) data = request.get_json() print(data) validate(data, alerts_schema) if data['alert_id']: # UPDATE alert_id = int(data['alert_id']) alert = obj_or_404(Alert.by_id(alert_id)) authz.require(alert.user_id == current_user.id) alert.query = data['query'] alert.label = data.get('custom_label', data['query']) alert.checking_interval=int(data.get('checking_interval', 9)) else: # CREATE alert = Alert( user_id = current_user.id, query=data['query'], label=data.get('custom_label', data['query']), checking_interval=int(data.get('checking_interval', 9)) ) db.session.add(alert) db.session.commit() return view(alert.id)
def view(alert_id): require(request.authz.logged_in) alert = obj_or_404(Alert.by_id(alert_id, role_id=request.authz.id)) return AlertSerializer.jsonify(alert)
def delete(alert_id): require(request.authz.session_write) alert = obj_or_404(Alert.by_id(alert_id, role_id=request.authz.id)) alert.delete() db.session.commit() return ('', 204)
def view(id): authz.require(authz.logged_in()) alert = obj_or_404(Alert.by_id(id)) authz.require(alert.user_id == current_user.id) return jsonify(alert)
def index(): if authz.logged_in(): alerts = Alert.by_role(request.auth_role).all() return jsonify({'results': alerts, 'total': len(alerts)}) return jsonify({'results': [], 'total': 0})
def view(id): enable_cache(vary_user=True) authz.require(authz.logged_in()) alert = obj_or_404(Alert.by_id(id, role=request.auth_role)) return jsonify(alert)
def index(): require(request.authz.logged_in) query = Alert.by_role_id(request.authz.id) result = DatabaseQueryResult(request, query) return AlertSerializer.jsonify_result(result)