Exemple #1
0
    def test_get_body(self):
        from flask import g
        with self.app.test_request_context('/'):
            g.login = '******'
            alert_in = Alert(
                resource='test1',
                event='event1',
                environment='Development',
                service=['svc1', 'svc2']
            )

            self.assertTrue(isinstance(alert_in.create_time, datetime))
            self.assertEqual(alert_in.last_receive_time, None)
            self.assertTrue(isinstance(alert_in.receive_time, datetime))
            self.assertEqual(alert_in.update_time, None)

            body = alert_in.get_body()
            self.assertEqual(type(body['createTime']), str)
            self.assertEqual(body['lastReceiveTime'], None)
            self.assertEqual(type(body['receiveTime']), str)
            self.assertEqual(body['updateTime'], None)

            alert_out = process_alert(alert_in)

            self.assertTrue(isinstance(alert_out.create_time, datetime))
            self.assertTrue(isinstance(alert_out.last_receive_time, datetime))
            self.assertTrue(isinstance(alert_out.receive_time, datetime))
            self.assertTrue(isinstance(alert_out.update_time, datetime))

            body = alert_out.get_body()
            self.assertEqual(type(body['createTime']), str)
            self.assertEqual(type(body['lastReceiveTime']), str)
            self.assertEqual(type(body['receiveTime']), str)
            self.assertEqual(type(body['updateTime']), str)
Exemple #2
0
def get_counts():
    query = qb.from_params(request.args)
    severity_count = Alert.get_counts_by_severity(query)
    status_count = Alert.get_counts_by_status(query)

    return jsonify(
        status="ok",
        total=sum(severity_count.values()),
        severityCounts=severity_count,
        statusCounts=status_count
    )
Exemple #3
0
def process_alert(alert: Alert) -> Alert:

    wanted_plugins, wanted_config = plugins.routing(alert)

    skip_plugins = False
    for plugin in wanted_plugins:
        if alert.is_suppressed:
            skip_plugins = True
            break
        try:
            alert = plugin.pre_receive(alert, config=wanted_config)
        except TypeError:
            alert = plugin.pre_receive(alert)  # for backward compatibility
        except (RejectException, HeartbeatReceived, BlackoutPeriod, RateLimit):
            raise
        except Exception as e:
            if current_app.config['PLUGINS_RAISE_ON_ERROR']:
                raise RuntimeError("Error while running pre-receive plugin '{}': {}".format(plugin.name, str(e)))
            else:
                logging.error("Error while running pre-receive plugin '{}': {}".format(plugin.name, str(e)))
        if not alert:
            raise SyntaxError("Plugin '%s' pre-receive hook did not return modified alert" % plugin.name)

    try:
        is_duplicate = alert.is_duplicate()
        if is_duplicate:
            alert = alert.deduplicate(is_duplicate)
        else:
            is_correlated = alert.is_correlated()
            if is_correlated:
                alert = alert.update(is_correlated)
            else:
                alert = alert.create()
    except Exception as e:
        raise ApiError(str(e))

    updated = None
    for plugin in wanted_plugins:
        if skip_plugins:
            break
        try:
            updated = plugin.post_receive(alert, config=wanted_config)
        except TypeError:
            updated = plugin.post_receive(alert)  # for backward compatibility
        except Exception as e:
            if current_app.config['PLUGINS_RAISE_ON_ERROR']:
                raise ApiError("Error while running post-receive plugin '{}': {}".format(plugin.name, str(e)))
            else:
                logging.error("Error while running post-receive plugin '{}': {}".format(plugin.name, str(e)))
        if updated:
            alert = updated

    if updated:
        alert.tag(alert.tags)
        alert.update_attributes(alert.attributes)

    return alert
Exemple #4
0
def housekeeping():
    DEFAULT_EXPIRED_DELETE_HRS = 2  # hours
    DEFAULT_INFO_DELETE_HRS = 12  # hours

    try:
        expired_threshold = int(request.args.get('expired', DEFAULT_EXPIRED_DELETE_HRS))
        info_threshold = int(request.args.get('info', DEFAULT_INFO_DELETE_HRS))
    except Exception as e:
        raise ApiError(str(e), 400)

    try:
        Alert.housekeeping(expired_threshold, info_threshold)
        return 'OK'
    except Exception as e:
        return 'HOUSEKEEPING FAILED: %s' % e, 503
Exemple #5
0
def pagerduty():

    data = request.json

    updated = False
    if data and 'messages' in data:
        for message in data['messages']:
            try:
                incident_key, status, text = parse_pagerduty(message)
            except ValueError as e:
                raise ApiError(str(e), 400)

            if not incident_key:
                raise ApiError('no incident key in PagerDuty data payload', 400)

            customer = g.get('customer', None)
            try:
                alert = Alert.find_by_id(id=incident_key, customer=customer)
            except Exception as e:
                raise ApiError(str(e), 500)

            if not alert:
                raise ApiError("not found", 404)

            try:
                updated = alert.set_status(status, text)
            except Exception as e:
                raise ApiError(str(e), 500)
    else:
        raise ApiError("no messages in PagerDuty data payload", 400)

    if updated:
        return jsonify(status="ok"), 200
    else:
        raise ApiError("update PagerDuty incident status failed", 500)
Exemple #6
0
def action_alerts(alerts: List[str], action: str, text: str, timeout: int) -> None:
    updated = []
    errors = []
    for alert_id in alerts:
        alert = Alert.find_by_id(alert_id)

        try:
            previous_status = alert.status
            alert, action, text = process_action(alert, action, text)
            alert = alert.from_action(action, text, timeout)
        except RejectException as e:
            errors.append(str(e))
            continue
        except InvalidAction as e:
            errors.append(str(e))
            continue
        except Exception as e:
            errors.append(str(e))
            continue

        if previous_status != alert.status:
            try:
                alert, status, text = process_status(alert, alert.status, text)
                alert = alert.from_status(status, text, timeout)
            except RejectException as e:
                errors.append(str(e))
                continue
            except Exception as e:
                errors.append(str(e))
                continue

        updated.append(alert.id)
Exemple #7
0
    def incoming(self, query_string, payload):

        if 'callback_query' in payload:
            author = payload['callback_query']['from']
            user = '******'.format(author.get('first_name'), author.get('last_name'))
            command, alert_id = payload['callback_query']['data'].split(' ', 1)

            customers = g.get('customers', None)
            alert = Alert.find_by_id(alert_id, customers=customers)
            if not alert:
                jsonify(status='error', message='alert not found for Telegram message')

            action = command.lstrip('/')
            if action in ['open', 'ack', 'close']:
                alert.set_status(status=action, text='status change via Telegram')
            elif action in ['watch', 'unwatch']:
                alert.untag(tags=['{}:{}'.format(action, user)])
            elif action == 'blackout':
                environment, resource, event = command.split('|', 2)
                blackout = Blackout(environment, resource=resource, event=event)
                blackout.create()

            send_message_reply(alert, action, user, payload)

            text = 'alert updated via telegram webhook'
            write_audit_trail.send(current_app._get_current_object(), event='webhook-updated', message=text,
                                   user=g.login, customers=g.customers, scopes=g.scopes, resource_id=alert.id,
                                   type='alert', request=request)

            return jsonify(status='ok')
        else:
            return jsonify(status='ok', message='no callback_query in Telegram message')
Exemple #8
0
    def incoming(self, query_string, payload):

        if payload and payload['state'] == 'alerting':
            return [parse_grafana(payload, match, query_string) for match in payload.get('evalMatches', [])]

        elif payload and payload['state'] == 'ok' and payload.get('ruleId'):
            try:
                query = qb.from_dict({'attributes.ruleId': str(payload['ruleId'])})
                existingAlerts = Alert.find_all(query)
            except Exception as e:
                raise ApiError(str(e), 500)

            alerts = []
            for updateAlert in existingAlerts:
                updateAlert.severity = 'normal'
                updateAlert.status = 'closed'

                try:
                    alert = process_alert(updateAlert)
                except RejectException as e:
                    raise ApiError(str(e), 403)
                except Exception as e:
                    raise ApiError(str(e), 500)
                alerts.append(alert)
            return alerts
        else:
            raise ApiError('no alerts in Grafana notification payload', 400)
Exemple #9
0
    def incoming(self, query_string, payload):

        updated = False
        if payload and 'messages' in payload:
            for message in payload['messages']:
                try:
                    incident_key, status, text = parse_pagerduty(message)
                except ValueError as e:
                    raise ApiError(str(e), 400)

                if not incident_key:
                    raise ApiError('no incident key in PagerDuty data payload', 400)

                customers = g.get('customers', None)
                try:
                    alert = Alert.find_by_id(id=incident_key, customers=customers)
                except Exception as e:
                    raise ApiError(str(e), 500)

                if not alert:
                    raise ApiError('not found', 404)

                try:
                    updated = alert.set_status(status, text)
                except Exception as e:
                    raise ApiError(str(e), 500)

            if updated:
                return jsonify(status='ok')
            else:
                raise ApiError('update PagerDuty incident status failed', 500)

        else:
            raise ApiError('no messages in PagerDuty data payload', 400)
Exemple #10
0
def receive():
    try:
        incomingAlert = Alert.parse(request.json)
    except ValueError as e:
        raise ApiError(str(e), 400)

    if g.get('customer', None):
        incomingAlert.customer = g.get('customer')

    add_remote_ip(request, incomingAlert)

    try:
        alert = process_alert(incomingAlert)
    except RejectException as e:
        raise ApiError(str(e), 403)
    except RateLimit as e:
        return jsonify(status="error", message=str(e), id=incomingAlert.id), 429
    except BlackoutPeriod as e:
        return jsonify(status="ok", message=str(e), id=incomingAlert.id), 202
    except Exception as e:
        raise ApiError(str(e), 500)

    if alert:
        return jsonify(status="ok", id=alert.id, alert=alert.serialize), 201
    else:
        raise ApiError("insert or update of received alert failed", 500)
Exemple #11
0
def process_action(alert: Alert, action: str, text: str) -> Tuple[Alert, str, str]:

    wanted_plugins, wanted_config = plugins.routing(alert)

    updated = None
    for plugin in wanted_plugins:
        if alert.is_suppressed:
            break
        try:
            updated = plugin.take_action(alert, action, text, config=wanted_config)
        except NotImplementedError:
            pass  # plugin does not support action() method
        except RejectException:
            raise
        except Exception as e:
            if current_app.config['PLUGINS_RAISE_ON_ERROR']:
                raise ApiError("Error while running action plugin '{}': {}".format(plugin.name, str(e)))
            else:
                logging.error("Error while running action plugin '{}': {}".format(plugin.name, str(e)))
        if updated:
            try:
                alert, action, text = updated
            except Exception:
                alert = updated

    # remove keys from attributes with None values
    new_attrs = {k: v for k, v in alert.attributes.items() if v is not None}
    alert.attributes = new_attrs

    return alert, action, text
Exemple #12
0
def telegram():

    data = request.json
    if 'callback_query' in data:
        author = data['callback_query']['from']
        user = "******".format(author.get('first_name'), author.get('last_name'))
        command, alert_id = data['callback_query']['data'].split(' ', 1)

        alert = Alert.find_by_id(alert_id)
        if not alert:
            jsonify(status="error", message="alert not found for Telegram message")

        action = command.lstrip('/')
        if action in ['open', 'ack', 'close']:
            alert.set_status(status=action, text='status change via Telegram')
        elif action in ['watch', 'unwatch']:
            alert.untag(tags=["{}:{}".format(action, user)])
        elif action == 'blackout':
            environment, resource, event = alert.split('|', 2)
            blackout = Blackout(environment, resource=resource, event=event)
            blackout.create()

        send_message_reply(alert, action, user, data)
        return jsonify(status="ok")
    else:
        return jsonify(status="error", message="no callback_query in Telegram message"), 400
Exemple #13
0
def bulk_set_status():
    status = request.json.get('status', None)
    text = request.json.get('text', 'bulk status update')
    timeout = request.json.get('timeout', None)

    if not status:
        raise ApiError("must supply 'status' as json data", 400)

    query = qb.from_params(request.args)
    alerts = Alert.find_all(query)

    if not alerts:
        raise ApiError('not found', 404)

    updated = []
    errors = []
    for alert in alerts:
        try:
            alert, status, text = process_status(alert, status, text)
        except RejectException as e:
            errors.append(str(e))
            continue
        except Exception as e:
            errors.append(str(e))
            continue

        if alert.set_status(status, text, timeout):
            updated.append(alert.id)

    if errors:
        raise ApiError('failed to bulk set alert status', 500, errors=errors)
    else:
        return jsonify(status='ok', updated=updated, count=len(updated))
Exemple #14
0
def process_status(alert: Alert, status: str, text: str) -> Tuple[Alert, str, str]:

    wanted_plugins, wanted_config = plugins.routing(alert)

    updated = None
    for plugin in wanted_plugins:
        if alert.is_suppressed:
            break
        try:
            updated = plugin.status_change(alert, status, text, config=wanted_config)
        except TypeError:
            updated = plugin.status_change(alert, status, text)  # for backward compatibility
        except RejectException:
            raise
        except Exception as e:
            if current_app.config['PLUGINS_RAISE_ON_ERROR']:
                raise ApiError("Error while running status plugin '{}': {}".format(plugin.name, str(e)))
            else:
                logging.error("Error while running status plugin '{}': {}".format(plugin.name, str(e)))
        if updated:
            try:
                alert, status, text = updated
            except Exception:
                alert = updated

    # remove keys from attributes with None values
    new_attrs = {k: v for k, v in alert.attributes.items() if v is not None}
    alert.attributes = new_attrs

    return alert, status, text
Exemple #15
0
def bulk_update_attributes():
    if not request.json.get('attributes', None):
        raise ApiError("must supply 'attributes' as json data", 400)

    query = qb.from_params(request.args)
    updated = Alert.update_attributes_find_all(query, request.json['attributes'])

    return jsonify(status='ok', updated=updated, count=len(updated))
Exemple #16
0
def bulk_untag_alert():
    if not request.json.get('tags', None):
        raise ApiError("must supply 'tags' as json list")

    query = qb.from_params(request.args)
    updated = Alert.untag_find_all(query, tags=request.json['tags'])

    return jsonify(status='ok', updated=updated, count=len(updated))
Exemple #17
0
def get_alert(alert_id):
    customer = g.get('customer', None)
    alert = Alert.find_by_id(alert_id, customer)

    if alert:
        return jsonify(status="ok", total=1, alert=alert.serialize)
    else:
        raise ApiError("not found", 404)
Exemple #18
0
def delete_alert(alert_id):
    customer = g.get('customer', None)
    alert = Alert.find_by_id(alert_id, customer)

    if not alert:
        raise ApiError("not found", 404)

    if alert.delete():
        return jsonify(status="ok")
    else:
        raise ApiError("failed to delete alert", 500)
Exemple #19
0
def history():
    query = qb.from_params(request.args, customers=g.customers)
    paging = Page.from_params(request.args, items=0)
    history = Alert.get_history(query, paging.page, paging.page_size)

    if history:
        return jsonify(status='ok',
                       history=[h.serialize for h in history],
                       total=len(history))
    else:
        return jsonify(status='ok', message='not found', history=[], total=0)
Exemple #20
0
def prometheus_metrics():

    total_alert_gauge.set(Alert.get_count())

    output = Gauge.find_all()
    output += Counter.find_all()
    output += Timer.find_all()

    return Response(
        [o.serialize(format='prometheus') for o in output],
        content_type='text/plain; version=0.0.4; charset=utf-8'
    )
Exemple #21
0
def receive():
    try:
        alert = Alert.parse(request.json)
    except ValueError as e:
        raise ApiError(str(e), 400)

    alert.customer = assign_customer(wanted=alert.customer)

    def audit_trail_alert(event: str):
        write_audit_trail.send(current_app._get_current_object(),
                               event=event,
                               message=alert.text,
                               user=g.login,
                               customers=g.customers,
                               scopes=g.scopes,
                               resource_id=alert.id,
                               type='alert',
                               request=request)

    try:
        alert = process_alert(alert)
    except RejectException as e:
        audit_trail_alert(event='alert-rejected')
        raise ApiError(str(e), 403)
    except RateLimit as e:
        audit_trail_alert(event='alert-rate-limited')
        return jsonify(status='error', message=str(e), id=alert.id), 429
    except HeartbeatReceived as heartbeat:
        audit_trail_alert(event='alert-heartbeat')
        return jsonify(status='ok', message=str(heartbeat),
                       id=heartbeat.id), 202
    except BlackoutPeriod as e:
        audit_trail_alert(event='alert-blackout')
        return jsonify(status='ok', message=str(e), id=alert.id), 202
    except ForwardingLoop as e:
        return jsonify(status='ok', message=str(e)), 202
    except Exception as e:
        raise ApiError(str(e), 500)

    write_audit_trail.send(current_app._get_current_object(),
                           event='alert-received',
                           message=alert.text,
                           user=g.login,
                           customers=g.customers,
                           scopes=g.scopes,
                           resource_id=alert.id,
                           type='alert',
                           request=request)

    if alert:
        return jsonify(status='ok', id=alert.id, alert=alert.serialize), 201
    else:
        raise ApiError('insert or update of received alert failed', 500)
Exemple #22
0
def prometheus_metrics():

    total_alert_gauge.set(Alert.get_count())

    output = Gauge.find_all()
    output += Counter.find_all()
    output += Timer.find_all()

    return Response(
        [o.serialize(format='prometheus') for o in output],
        content_type='text/plain; version=0.0.4; charset=utf-8'
    )
Exemple #23
0
def status():

    now = int(time.time() * 1000)
    total_alert_gauge.set(Alert.get_count())

    metrics = Gauge.find_all()
    metrics.extend(Counter.find_all())
    metrics.extend(Timer.find_all())
    metrics.extend(Switch.find_all())

    return jsonify(application="alerta", version=__version__, time=now, uptime=int(now - started),
                   metrics=[metric.serialize() for metric in metrics])
Exemple #24
0
def status():

    now = int(time.time() * 1000)
    total_alert_gauge.set(Alert.get_count())

    metrics = Gauge.find_all()
    metrics.extend(Counter.find_all())
    metrics.extend(Timer.find_all())
    metrics.extend(Switch.find_all())

    return jsonify(application="alerta", version=__version__, time=now, uptime=int(now - started),
                   metrics=[metric.serialize() for metric in metrics])
Exemple #25
0
def search_alerts():
    query_time = datetime.utcnow()
    query = qb.from_params(request.args, query_time)
    severity_count = Alert.get_counts_by_severity(query)
    status_count = Alert.get_counts_by_status(query)

    total = sum(severity_count.values())
    paging = Page.from_params(request.args, total)

    alerts = Alert.find_all(query, paging.page, paging.page_size)

    if alerts:
        return jsonify(
            status="ok",
            page=paging.page,
            pageSize=paging.page_size,
            pages=paging.pages,
            more=paging.has_more,
            alerts=[alert.serialize for alert in alerts],
            total=total,
            statusCounts=status_count,
            severityCounts=severity_count,
            lastTime=max([alert.last_receive_time for alert in alerts]),
            autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on
        )
    else:
        return jsonify(
            status="ok",
            message="not found",
            page=paging.page,
            pageSize=paging.page_size,
            pages=0,
            more=False,
            alerts=[],
            total=0,
            severityCounts=severity_count,
            statusCounts=status_count,
            lastTime=query_time,
            autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on
        )
Exemple #26
0
def search_alerts():
    query_time = datetime.utcnow()
    query = qb.from_params(request.args,
                           customers=g.customers,
                           query_time=query_time)
    severity_count = Alert.get_counts_by_severity(query)
    status_count = Alert.get_counts_by_status(query)

    total = sum(severity_count.values())
    paging = Page.from_params(request.args, total)

    alerts = Alert.find_all(query, paging.page, paging.page_size)

    if alerts:
        return jsonify(
            status='ok',
            page=paging.page,
            pageSize=paging.page_size,
            pages=paging.pages,
            more=paging.has_more,
            alerts=[alert.serialize for alert in alerts],
            total=total,
            statusCounts=status_count,
            severityCounts=severity_count,
            lastTime=max([alert.last_receive_time for alert in alerts]),
            autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on)
    else:
        return jsonify(
            status='ok',
            message='not found',
            page=paging.page,
            pageSize=paging.page_size,
            pages=0,
            more=False,
            alerts=[],
            total=0,
            severityCounts=severity_count,
            statusCounts=status_count,
            lastTime=query_time,
            autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on)
Exemple #27
0
    def incoming(self, query_string, payload):

        incident = payload['incident']
        state = incident['state']

        # 'documentation' is an optional field that you can use to customize
        # your alert sending a json
        if 'documentation' in incident:
            try:
                content = json.loads(incident['documentation']['content'])
                incident.update(content)
            except Exception:
                current_app.logger.warning("Invalid documentation content: '{}'".format(incident['documentation']))

        service = []
        status = None
        create_time = None  # type: ignore
        severity = incident.get('severity', 'critical')

        if incident['policy_name']:
            service.append(incident['policy_name'])

        if state == 'open':
            create_time = datetime.utcfromtimestamp(incident['started_at'])
        elif state == 'acknowledged':
            status = 'ack'
        elif state == 'closed':
            severity = 'ok'
            create_time = datetime.utcfromtimestamp(incident['ended_at'])
        else:
            severity = 'indeterminate'

        return Alert(
            resource=incident['resource_name'],
            event=incident['condition_name'],
            environment=incident.get('environment', 'Production'),
            severity=severity,
            status=status,
            service=service,
            group=incident.get('group', 'Cloud'),
            text=incident['summary'],
            attributes={
                'incidentId': incident['incident_id'],
                'resourceId': incident['resource_id'],
                'moreInfo': '<a href="%s" target="_blank">Stackdriver Console</a>' % incident['url']
            },
            customer=incident.get('customer'),
            origin=incident.get('origin', 'Stackdriver'),
            event_type='stackdriverAlert',
            create_time=create_time,
            raw_data=payload
        )
Exemple #28
0
    def incoming(self, path, query_string, payload):

        if 'version' not in payload:
            raise ValueError('New Relic Legacy Alerting is not supported')

        status = payload['current_state'].lower()
        if status == 'open':
            severity = payload['severity'].lower()
        elif status == 'acknowledged':
            severity = payload['severity'].lower()
            status = 'ack'
        elif status == 'closed':
            severity = 'ok'
        elif payload['severity'].lower() == 'info':
            severity = 'informational'
            status = 'open'
        else:
            severity = payload['severity'].lower()
            status = 'open'

        if severity not in SEVERITY_MAP:
            if severity.lower() == 'info':
                severity = 'informational'
            else:
                severity = 'unknown'

        attributes = dict()
        if 'incident_url' in payload and payload['incident_url'] is not None:
            attributes['incident_url'] = payload['incident_url']
        if 'runbook_url' in payload and payload['runbook_url'] is not None:
            attributes['runbook_url'] = payload['runbook_url']

        resource = payload['targets'][0]['name'] or UNKNOWN
        event = payload['condition_name'] or UNKNOWN

        return Alert(
            resource=resource,
            event=event,
            environment='Production',
            severity=severity,
            status=status,
            service=[payload['account_name']],
            group=payload['targets'][0]['type'],
            text=payload['details'],
            tags=[
                '{}:{}'.format(key, value)
                for (key, value) in payload['targets'][0]['labels'].items()
            ],
            attributes=attributes,
            origin='New Relic/v%s' % payload['version'],
            event_type=payload['event_type'].lower(),
            raw_data=payload)
Exemple #29
0
def grafana():

    alerts = []
    data = request.json
    if data and data['state'] == 'alerting':
        for match in data.get('evalMatches', []):
            try:
                incomingAlert = parse_grafana(data, match, request.args)
            except ValueError as e:
                return jsonify(status='error', message=str(e)), 400

            incomingAlert.customer = assign_customer(wanted=incomingAlert.customer)
            add_remote_ip(request, incomingAlert)

            try:
                alert = process_alert(incomingAlert)
            except RejectException as e:
                return jsonify(status='error', message=str(e)), 403
            except Exception as e:
                return jsonify(status='error', message=str(e)), 500
            alerts.append(alert)

    elif data and data['state'] == 'ok' and data.get('ruleId', None):
        try:
            query = qb.from_dict({'attributes.ruleId': str(data['ruleId'])})
            existingAlerts = Alert.find_all(query)
        except Exception as e:
            raise ApiError(str(e), 500)

        for updateAlert in existingAlerts:
            updateAlert.severity = 'normal'
            updateAlert.status = 'closed'

            try:
                alert = process_alert(updateAlert)
            except RejectException as e:
                raise ApiError(str(e), 403)
            except Exception as e:
                raise ApiError(str(e), 500)
            alerts.append(alert)
    else:
        raise ApiError('no alerts in Grafana notification payload', 400)

    for alert in alerts:
        text = 'grafana alert received via webhook'
        write_audit_trail.send(current_app._get_current_object(), event='webhook-received', message=text, user=g.user,
                               customers=g.customers, scopes=g.scopes, resource_id=alert.id, type='alert', request=request)

    if len(alerts) == 1:
        return jsonify(status='ok', id=alerts[0].id, alert=alerts[0].serialize), 201
    else:
        return jsonify(status='ok', ids=[alert.id for alert in alerts]), 201
Exemple #30
0
def action_alert(alert_id):
    action = request.json.get('action', None)
    text = request.json.get('text', f'{action} operator action')
    timeout = request.json.get('timeout', None)

    if not action:
        raise ApiError("must supply 'action' as json data", 400)

    customers = g.get('customers', None)
    alert = Alert.find_by_id(alert_id, customers)

    if not alert:
        raise ApiError('not found', 404)

    try:
        alert, action, text, timeout = process_action(alert, action, text,
                                                      timeout)
        alert = alert.from_action(action, text, timeout)
    except RejectException as e:
        write_audit_trail.send(current_app._get_current_object(),
                               event='alert-action-rejected',
                               message=alert.text,
                               user=g.login,
                               customers=g.customers,
                               scopes=g.scopes,
                               resource_id=alert.id,
                               type='alert',
                               request=request)
        raise ApiError(str(e), 400)
    except InvalidAction as e:
        raise ApiError(str(e), 409)
    except ForwardingLoop as e:
        return jsonify(status='ok', message=str(e)), 202
    except AlertaException as e:
        raise ApiError(e.message, code=e.code, errors=e.errors)
    except Exception as e:
        raise ApiError(str(e), 500)

    write_audit_trail.send(current_app._get_current_object(),
                           event='alert-actioned',
                           message=text,
                           user=g.login,
                           customers=g.customers,
                           scopes=g.scopes,
                           resource_id=alert.id,
                           type='alert',
                           request=request)

    if alert:
        return jsonify(status='ok')
    else:
        raise ApiError('failed to action alert', 500)
Exemple #31
0
    def incoming(self, path, query_string, payload):

        incident = payload['incident']
        state = incident['state']

        # 'documentation' is an optional field that you can use to customize
        # your alert sending a json
        if 'documentation' in incident:
            try:
                content = json.loads(incident['documentation']['content'])
                incident.update(content)
            except Exception:
                current_app.logger.warning(f"Invalid documentation content: '{incident['documentation']}'")

        status = None
        severity = incident.get('severity', 'critical')

        if state == 'open':
            status = None
        elif state == 'acknowledged':
            status = 'ack'
        elif state == 'closed':
            severity = 'ok'
        else:
            severity = 'indeterminate'

        service = []
        if incident['policy_name']:
            service.append(incident['policy_name'])

        return Alert(
            resource=incident['resource_name'],
            event=incident['condition_name'],
            environment=incident.get('environment', current_app.config['DEFAULT_ENVIRONMENT']),
            severity=severity,
            status=status,
            service=service,
            group=incident.get('group', 'Cloud'),
            text=incident['summary'],
            attributes={
                'incidentId': incident['incident_id'],
                'resourceId': incident['resource_id'],
                'moreInfo': f"<a href=\"{incident['url']}\" target=\"_blank\">Stackdriver Console</a>",
                'startedAt': incident['started_at'],
                'endedAt': incident['ended_at']

            },
            customer=incident.get('customer'),
            origin=incident.get('origin', 'Stackdriver'),
            event_type='stackdriverAlert',
            raw_data=payload
        )
Exemple #32
0
def set_customer(alert_id):
    want_customer = request.json.get('customer', None)
    query = qb.from_params(MultiDict([]), customers=g.customers)
    query_data = [
        c for c in Customer.find_all(query) if Scope.admin in g.scopes
        or Scope.admin_customers in g.scopes or c.customer in g.customers
    ]
    if query_data:
        list_customers = [q.serialize for q in query_data]
    else:
        raise ApiError('not found any customer ', 404)
    #LOG.info(list_customers)
    found = False
    for c in list_customers:
        if want_customer == c['customer']:
            found = True

    if found == False:
        raise ApiError('not found customer ', 404)

    alert = Alert.find_by_id(alert_id, g.get('customers', None))
    if not alert:
        raise ApiError('not found alert', 404)
    try:
        alert = alert.from_customer(want_customer)
    except RejectException as e:
        write_audit_trail.send(current_app._get_current_object(),
                               event='alert-customer-change-rejected',
                               message=alert.text,
                               user=g.login,
                               customers=g.customers,
                               scopes=g.scopes,
                               resource_id=alert.id,
                               type='alert',
                               request=request)
        raise ApiError(str(e), 400)
    except Exception as e:
        raise ApiError(str(e), 500)

    write_audit_trail.send(current_app._get_current_object(),
                           event='alert-customer-changed',
                           message='change customer to' + want_customer,
                           user=g.login,
                           customers=g.customers,
                           scopes=g.scopes,
                           resource_id=alert.id,
                           type='alert',
                           request=request)
    if alert:
        return jsonify(status='ok')
    else:
        raise ApiError('failed to set customer', 500)
Exemple #33
0
    def test_invalid(self):

        with self.app.test_request_context('/'):
            self.app.preprocess_request()
            with self.assertRaises(Exception) as e:
                process_alert(
                    Alert(resource='foo',
                          event='bar',
                          environment='Development',
                          service=['Svc'],
                          severity='baz'))
            exc = e.exception
            self.assertEqual(str(exc), '\'baz\' is not a valid severity')
Exemple #34
0
def process_status(alert: Alert, status: str,
                   text: str) -> Tuple[Alert, str, str]:

    wanted_plugins, wanted_config = plugins.routing(alert)

    updated = None
    for plugin in wanted_plugins:
        if alert.is_suppressed:
            break
        try:
            updated = plugin.status_change(alert,
                                           status,
                                           text,
                                           config=wanted_config)
        except TypeError:
            updated = plugin.status_change(alert, status,
                                           text)  # for backward compatibility
        except (RejectException, AlertaException):
            raise
        except Exception as e:
            if current_app.config['PLUGINS_RAISE_ON_ERROR']:
                raise ApiError(
                    f"Error while running status plugin '{plugin.name}': {str(e)}"
                )
            else:
                logging.error(
                    f"Error while running status plugin '{plugin.name}': {str(e)}"
                )
        if updated:
            try:
                alert, status, text = updated
            except Exception:
                alert = updated

    if updated:
        alert.update_tags(alert.tags)
        alert.attributes = alert.update_attributes(alert.attributes)

    return alert, status, text
Exemple #35
0
def get_environments():
    query = qb.from_params(request.args)
    environments = Alert.get_environments(query)

    if environments:
        return jsonify(status="ok",
                       environments=environments,
                       total=len(environments))
    else:
        return jsonify(status="ok",
                       message="not found",
                       environments=[],
                       total=0)
Exemple #36
0
def get_environments():
    query = qb.from_params(request.args, customers=g.customers)
    environments = Alert.get_environments(query)

    if environments:
        return jsonify(status='ok',
                       environments=environments,
                       total=len(environments))
    else:
        return jsonify(status='ok',
                       message='not found',
                       environments=[],
                       total=0)
Exemple #37
0
def parse_riemann(alert):

    return Alert(resource='%s-%s' % (alert['host'], alert['service']),
                 event=alert.get('event', alert['service']),
                 environment=alert.get('environment', 'Production'),
                 severity=alert.get('state', 'unknown'),
                 service=[alert['service']],
                 group=alert.get('group', 'Performance'),
                 text=alert.get('description', None),
                 value=alert.get('metric', None),
                 tags=alert.get('tags', None),
                 origin='Riemann',
                 raw_data=alert)
Exemple #38
0
def parse_stackdriver(notification: JSON) -> Alert:

    incident = notification['incident']
    state = incident['state']

    # 'documentation' is an optional field that you can use to customize
    # your alert sending a json
    if 'documentation' in incident:
        try:
            content = json.loads(incident['documentation']['content'])
            incident.update(content)
        except Exception as e:
            LOG.warning("Invalid documentation content: '{}'".format(incident['documentation']))

    if state == 'open':
        severity = 'critical'
        status = None
        create_time = datetime.utcfromtimestamp(incident['started_at'])
    elif state == 'acknowledged':
        severity = 'critical'
        status = 'ack'
        create_time = None  # type: ignore
    elif state == 'closed':
        severity = 'ok'
        status = None
        create_time = datetime.utcfromtimestamp(incident['ended_at'])
    else:
        severity = 'indeterminate'
        status = None
        create_time = None  # type: ignore

    return Alert(
        resource=incident['resource_name'],
        event=incident['condition_name'],
        environment='Production',
        severity=severity,
        status=status,
        service=[incident['policy_name']],
        group='Cloud',
        text=incident['summary'],
        attributes={
            'incidentId': incident['incident_id'],
            'resourceId': incident['resource_id'],
            'moreInfo': '<a href="%s" target="_blank">Stackdriver Console</a>' % incident['url']
        },
        customer=incident.get('customer'),
        origin='Stackdriver',
        event_type='stackdriverAlert',
        create_time=create_time,
        raw_data=notification
    )
Exemple #39
0
def add_note(alert_id):
    note_text = request.json.get('text') or request.json.get('note')

    if not note_text:
        raise ApiError("must supply 'note' text", 400)

    customers = g.get('customers', None)
    alert = Alert.find_by_id(alert_id, customers)

    if not alert:
        raise ApiError('not found', 404)

    try:
        alert, note_text = process_note(alert, note_text)
        note = alert.add_note(note_text)
    except RejectException as e:
        write_audit_trail.send(current_app._get_current_object(),
                               event='alert-note-rejected',
                               message='',
                               user=g.login,
                               customers=g.customers,
                               scopes=g.scopes,
                               resource_id=note.id,
                               type='note',
                               request=request)
        raise ApiError(str(e), 400)
    except ForwardingLoop as e:
        return jsonify(status='ok', message=str(e)), 202
    except AlertaException as e:
        raise ApiError(e.message, code=e.code, errors=e.errors)
    except Exception as e:
        raise ApiError(str(e), 500)

    write_audit_trail.send(current_app._get_current_object(),
                           event='alert-note-added',
                           message='',
                           user=g.login,
                           customers=g.customers,
                           scopes=g.scopes,
                           resource_id=note.id,
                           type='note',
                           request=request)

    if note:
        return jsonify(status='ok', id=note.id, note=note.serialize), 201, {
            'Location':
            absolute_url('/alert/{}/note/{}'.format(alert.id, note.id))
        }
    else:
        raise ApiError('failed to add note for alert', 500)
Exemple #40
0
def update_attributes(alert_id):
    if not request.json.get('attributes', None):
        raise ApiError("must supply 'attributes' as json data", 400)

    customer = g.get('customer', None)
    alert = Alert.find_by_id(alert_id, customer)

    if not alert:
        raise ApiError("not found", 404)

    if alert.update_attributes(request.json['attributes']):
        return jsonify(status="ok")
    else:
        raise ApiError("failed to update attributes", 500)
Exemple #41
0
def untag_alert(alert_id):
    if not request.json.get('tags', None):
        raise ApiError("must supply 'tags' as json list")

    customer = g.get('customer', None)
    alert = Alert.find_by_id(alert_id, customer)

    if not alert:
        raise ApiError("not found", 404)

    if alert.untag(tags=request.json['tags']):
        return jsonify(status="ok")
    else:
        raise ApiError("failed to untag alert", 500)
Exemple #42
0
def tag_alert(alert_id):
    if not request.json.get('tags', None):
        raise ApiError("must supply 'tags' as json list")

    customers = g.get('customers', None)
    alert = Alert.find_by_id(alert_id, customers)

    if not alert:
        raise ApiError('not found', 404)

    if alert.tag(tags=request.json['tags']):
        return jsonify(status='ok')
    else:
        raise ApiError('failed to tag alert', 500)
Exemple #43
0
def grafana():

    alerts = []
    data = request.json
    if data and data['state'] == 'alerting':
        for match in data.get('evalMatches', []):
            try:
                incomingAlert = parse_grafana(data, match)
            except ValueError as e:
                return jsonify(status="error", message=str(e)), 400

            if g.get('customer', None):
                incomingAlert.customer = g.get('customer')

            add_remote_ip(request, incomingAlert)

            try:
                alert = process_alert(incomingAlert)
            except RejectException as e:
                return jsonify(status="error", message=str(e)), 403
            except Exception as e:
                return jsonify(status="error", message=str(e)), 500
            alerts.append(alert)

    elif data and data['state'] == 'ok' and data.get('ruleId', None):
        try:
            query = qb.from_dict({'attributes.ruleId': str(data['ruleId'])})
            existingAlerts = Alert.find_all(query)
        except Exception as e:
            raise ApiError(str(e), 500)

        for updateAlert in existingAlerts:
            updateAlert.severity = 'normal'
            updateAlert.status = 'closed'

            try:
                alert = process_alert(updateAlert)
            except RejectException as e:
                raise ApiError(str(e), 403)
            except Exception as e:
                raise ApiError(str(e), 500)
            alerts.append(alert)
    else:
        raise ApiError("no alerts in Grafana notification payload", 400)

    if len(alerts) == 1:
        return jsonify(status="ok", id=alerts[0].id,
                       alert=alerts[0].serialize), 201
    else:
        return jsonify(status="ok", ids=[alert.id for alert in alerts]), 201
Exemple #44
0
 def incoming(self, path, query_string, payload):
     if payload['criticality'] == 'ALERT_CRITICALITY_LEVEL_WARNING':
         severity = 'critical'
     else:
         severity = 'normal'
     return Alert(id=payload['alertId'],
                  resource=payload['resourceName'],
                  event=payload['subType'],
                  environment='Production',
                  service=[payload['resourceKind']],
                  severity=severity,
                  group=payload['resourceKind'],
                  type=payload['type'],
                  text=payload['info'])
Exemple #45
0
    def incoming(self, query_string, payload):

        return Alert(resource='{}-{}'.format(payload['host'],
                                             payload['service']),
                     event=payload.get('event', payload['service']),
                     environment=payload.get('environment', 'Production'),
                     severity=payload.get('state', 'unknown'),
                     service=[payload['service']],
                     group=payload.get('group', 'Performance'),
                     text=payload.get('description', None),
                     value=payload.get('metric', None),
                     tags=payload.get('tags', None),
                     origin='Riemann',
                     raw_data=payload)
Exemple #46
0
def parse_graylog(alert):

    return Alert(
        resource=alert['stream']['title'],
        event="Alert",
        environment='Development',
        service=["test"],
        severity="critical",
        value="n/a",
        text=alert['check_result']['result_description'],
        attributes={'checkId': alert['check_result']['triggered_condition']['id']},
        origin='Graylog',
        event_type='performanceAlert',
        raw_data=alert)
Exemple #47
0
def delete_alert(alert_id):
    customers = g.get('customers', None)
    alert = Alert.find_by_id(alert_id, customers)

    if not alert:
        raise ApiError('not found', 404)

    write_audit_trail.send(current_app._get_current_object(), event='alert-deleted', message='', user=g.user,
                           customers=g.customers, scopes=g.scopes, resource_id=alert.id, type='alert', request=request)

    if alert.delete():
        return jsonify(status='ok')
    else:
        raise ApiError('failed to delete alert', 500)
Exemple #48
0
def parse_grafana(alert, match, args):
    alerting_severity = args.get('severity', 'major')
    if alerting_severity not in Severity.SEVERITY_MAP:
        raise ValueError('Invalid severity parameter, expected one of %s' %
                         ', '.join(sorted(Severity.SEVERITY_MAP)))

    if alert['state'] == 'alerting':
        severity = alerting_severity
    elif alert['state'] == 'ok':
        severity = 'normal'
    else:
        severity = 'indeterminate'

    environment = args.get('environment',
                           'Production')  # TODO: verify at create?
    event_type = args.get('event_type', 'performanceAlert')
    group = args.get('group', 'Performance')
    origin = args.get('origin', 'Grafana')
    service = args.get('service', 'Grafana')
    timeout = args.get('timeout', current_app.config['ALERT_TIMEOUT'])

    attributes = match.get('tags', None) or dict()
    attributes = {k.replace('.', '_'): v for (k, v) in attributes.items()}

    attributes['ruleId'] = str(alert['ruleId'])
    if 'ruleUrl' in alert:
        attributes[
            'ruleUrl'] = '<a href="%s" target="_blank">Rule</a>' % alert[
                'ruleUrl']
    if 'imageUrl' in alert:
        attributes[
            'imageUrl'] = '<a href="%s" target="_blank">Image</a>' % alert[
                'imageUrl']

    return Alert(resource=match['metric'],
                 event=alert['ruleName'],
                 environment=environment,
                 severity=severity,
                 service=[service],
                 group=group,
                 value='%s' % match['value'],
                 text=alert.get('message', None)
                 or alert.get('title', alert['state']),
                 tags=list(),
                 attributes=attributes,
                 origin=origin,
                 event_type=event_type,
                 timeout=timeout,
                 raw_data=json.dumps(alert))
Exemple #49
0
def grafana():

    alerts = []
    data = request.json
    if data and data['state'] == 'alerting':
        for match in data.get('evalMatches', []):
            try:
                incomingAlert = parse_grafana(data, match)
            except ValueError as e:
                return jsonify(status="error", message=str(e)), 400

            if g.get('customer', None):
                incomingAlert.customer = g.get('customer')

            add_remote_ip(request, incomingAlert)

            try:
                alert = process_alert(incomingAlert)
            except RejectException as e:
                return jsonify(status="error", message=str(e)), 403
            except Exception as e:
                return jsonify(status="error", message=str(e)), 500
            alerts.append(alert)

    elif data and data['state'] == 'ok' and data.get('ruleId', None):
        try:
            query = qb.from_dict({'attributes.ruleId': str(data['ruleId'])})
            existingAlerts = Alert.find_all(query)
        except Exception as e:
            raise ApiError(str(e), 500)

        for updateAlert in existingAlerts:
            updateAlert.severity = 'normal'
            updateAlert.status = 'closed'

            try:
                alert = process_alert(updateAlert)
            except RejectException as e:
                raise ApiError(str(e), 403)
            except Exception as e:
                raise ApiError(str(e), 500)
            alerts.append(alert)
    else:
        raise ApiError("no alerts in Grafana notification payload", 400)

    if len(alerts) == 1:
        return jsonify(status="ok", id=alerts[0].id, alert=alerts[0].serialize), 201
    else:
        return jsonify(status="ok", ids=[alert.id for alert in alerts]), 201
Exemple #50
0
    def incoming(self, query_string, payload):

        if 'environment' not in payload:
            raise ValueError('Environment must be set.')

        status = payload['state'].lower()
        if status == '0':
            severity = payload['severity'].lower()
        elif status == '1':
            severity = payload['severity'].lower()
            status = 'ack'
        elif status == '4':
            severity = 'ok'
        elif status == '5':
            severity = 'cleared'
        elif payload['severity'].lower() == 'info':
            severity = 'informational'
            status = 'open'
        else:
            severity = payload['severity'].lower()
            status = 'open'

        attributes = dict()
        if 'incident_url' in payload:
            attributes[
                'event_url'] = '<a href="%s" target="_blank">Event URL</a>' % payload[
                    'event_url']
        if 'runbook_url' in payload:
            attributes[
                'runBook'] = '<a href="%s" target="_blank">Runbook URL</a>' % payload[
                    'runbook_url']

        return Alert(
            resource=payload['resource'],
            event=payload['event'],
            environment='Production',
            severity=severity,
            status=status,
            service=[payload['service']],
            group=payload['group'],
            text=payload['test'],
            tags=[
                '{}:{}'.format(key, value)
                for (key, value) in payload['targets'][0]['labels'].items()
            ],
            attributes=attributes,
            origin=payload['origin'],
            event_type=payload['event_type'].lower(),
            raw_data=payload)
Exemple #51
0
def get_notes(alert_id):
    customers = g.get('customers', None)
    alert = Alert.find_by_id(alert_id, customers)

    if not alert:
        raise ApiError('not found', 404)

    notes = alert.get_alert_notes()

    if notes:
        return jsonify(status='ok',
                       notes=[note.serialize for note in notes],
                       total=len(notes))
    else:
        return jsonify(status='ok', message='not found', notes=[], total=0)
Exemple #52
0
    def incoming(self, path, query_string, payload):

        return Alert(
            resource=f"{payload['host']}-{payload['service']}",
            event=payload.get('event', payload['service']),
            environment=payload.get('environment', current_app.config['DEFAULT_ENVIRONMENT']),
            severity=payload.get('state', 'unknown'),
            service=[payload['service']],
            group=payload.get('group', 'Performance'),
            text=payload.get('description', None),
            value=payload.get('metric', None),
            tags=payload.get('tags', None),
            origin='Riemann',
            raw_data=payload
        )
Exemple #53
0
def slack():
    alert_id, user, action = parse_slack(request.form)

    alert = Alert.find_by_id(alert_id)
    if not alert:
        jsonify(status="error", message="alert not found for #slack message")

    if action in ['open', 'ack', 'close']:
        alert.set_status(status=action, text="status change via #slack by {}".format(user))
    elif action in ['watch', 'unwatch']:
        alert.untag(alert.id, ["{}:{}".format(action, user)])
    else:
        raise ApiError('Unsupported #slack action', 400)

    response = build_slack_response(alert, action, user, request.form)
    return jsonify(**response), 201
Exemple #54
0
def send_message_reply(alert: Alert, action: str, user: str, data: JSON) -> None:
    try:
        import telepot  # type: ignore
    except ImportError as e:
        current_app.logger.warning("You have configured Telegram but 'telepot' client is not installed", exc_info=True)
        return

    try:
        bot_id = os.environ.get('TELEGRAM_TOKEN') or current_app.config.get('TELEGRAM_TOKEN')
        dashboard_url = os.environ.get('DASHBOARD_URL') or current_app.config.get('DASHBOARD_URL')
        chat_id = os.environ.get('TELEGRAM_CHAT_ID') or current_app.config.get('TELEGRAM_CHAT_ID')
        bot = telepot.Bot(bot_id)

        # message info
        message_id = data['callback_query']['message']['message_id']
        message_log = '\n'.join(data['callback_query']['message']['text'].split('\n')[1:])

        # process buttons for reply text
        inline_keyboard, reply = [], 'The status of alert {alert} is *{status}* now!'  # type: List[List[JSON]], str

        actions = ['watch', 'unwatch']
        if action in actions:
            reply = 'User `{user}` is _{status}ing_ alert {alert}'
            next_action = actions[(actions.index(action) + 1) % len(actions)]
            inline_keyboard = [
                [
                    {'text': next_action.capitalize(), 'callback_data': '/{} {}'.format(next_action, alert.id)},
                    {'text': 'Ack', 'callback_data': '{} {}'.format('/ack', alert.id)},
                    {'text': 'Close', 'callback_data': '{} {}'.format('/close', alert.id)}
                ]
            ]

        # format message response
        alert_short_id = alert.get_id(short=True)
        alert_url = '{}/#/alert/{}'.format(dashboard_url, alert.id)
        reply = reply.format(alert=alert_short_id, status=action, user=user)
        message = '{alert} *{level} - {event} on {resouce}*\n{log}\n{reply}'.format(
            alert='[{}]({})'.format(alert_short_id, alert_url), level=alert.severity.capitalize(),
            event=alert.event, resouce=alert.resource, log=message_log, reply=reply)

        # send message
        bot.editMessageText(
            msg_identifier=(chat_id, message_id), text=message,
            parse_mode='Markdown', reply_markup={'inline_keyboard': inline_keyboard}
        )
    except Exception as e:
        current_app.logger.warning('Error sending reply message', exc_info=True)
Exemple #55
0
def get_services():
    query = qb.from_params(request.args)
    services = Alert.get_services(query)

    if services:
        return jsonify(
            status="ok",
            services=services,
            total=len(services)
        )
    else:
        return jsonify(
            status="ok",
            message="not found",
            services=[],
            total=0
        )
Exemple #56
0
def get_tags():
    query = qb.from_params(request.args)
    tags = Alert.get_tags(query)

    if tags:
        return jsonify(
            status="ok",
            tags=tags,
            total=len(tags)
        )
    else:
        return jsonify(
            status="ok",
            message="not found",
            tags=[],
            total=0
        )
Exemple #57
0
def get_environments():
    query = qb.from_params(request.args)
    environments = Alert.get_environments(query)

    if environments:
        return jsonify(
            status="ok",
            environments=environments,
            total=len(environments)
        )
    else:
        return jsonify(
            status="ok",
            message="not found",
            environments=[],
            total=0
        )
Exemple #58
0
def get_top10_flapping():
    query = qb.from_params(request.args)
    top10 = Alert.get_top10_flapping(query)

    if top10:
        return jsonify(
            status="ok",
            top10=top10,
            total=len(top10)
        )
    else:
        return jsonify(
            status="ok",
            message="not found",
            top10=[],
            total=0
        )