Esempio n. 1
0
    def __init__(self, config, namespace='', quit_check_callback=None):
        super().__init__(config, namespace=namespace, quit_check_callback=quit_check_callback)

        self.es_context = self.config.elasticsearch.elasticsearch_class(
            config=self.config.elasticsearch
        )
        self.metrics = markus.get_metrics(namespace)
Esempio n. 2
0
def setup_metrics(config, local_unused, args_unused):
    """Sets up markus and adds a metrics client to config

    :returns: Markus MetricsInterface

    """
    backends = []

    for backend in config.metricscfg.markus_backends:
        if backend == 'markus.backends.statsd.StatsdMetrics':
            backends.append({
                'class': 'markus.backends.statsd.StatsdMetrics',
                'options': {
                    'statsd_host': config.metricscfg.statsd_host,
                    'statsd_port': config.metricscfg.statsd_port,
                }
            })
        elif backend == 'markus.backends.datadog.DatadogMetrics':
            backends.append({
                'class': 'markus.backends.datadog.DatadogMetrics',
                'options': {
                    'statsd_host': config.metricscfg.statsd_host,
                    'statsd_port': config.metricscfg.statsd_port,
                }
            })
        elif backend == 'markus.backends.logging.LoggingMetrics':
            backends.append({
                'class': 'markus.backends.logging.LoggingMetrics',
            })
        else:
            raise ValueError('Invalid markus backend "%s"' % backend)

    markus.configure(backends=backends)

    return markus.get_metrics('')
Esempio n. 3
0
    def __init__(self, config):
        super().__init__(config)
        self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL)
        self.metrics = markus.get_metrics('processor.betaversionrule')

        # For looking up version strings
        self.version_string_api = config.version_string_api
        self.session = session_with_retries()
Esempio n. 4
0
    def __init__(self, config):
        super().__init__(config)
        self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL)
        self.metrics = markus.get_metrics('processor.betaversionrule')

        # For looking up version strings
        self.version_string_api = config.version_string_api
        self.session = session_with_retries()
Esempio n. 5
0
 def __init__(self, config, quit_check_callback=None):
     self.config = config
     self._CreateError = boto.exception.StorageCreateError
     self.ResponseError = (boto.exception.StorageResponseError, KeyNotFound)
     self._bucket_cache = {}
     self.metrics = markus.get_metrics(config.boto_metrics_prefix)
     self._connect_to_endpoint = boto.connect_s3
     self._calling_format = config.calling_format
Esempio n. 6
0
def test_timer_contextmanager(metricsmock):
    metrics = get_metrics("thing")

    with metricsmock as mm:
        with metrics.timer("long_fun"):
            print("blah")

    assert mm.has_record(fun_name="timing", stat="thing.long_fun")
Esempio n. 7
0
    def __init__(self, config, namespace="", quit_check_callback=None):
        super().__init__(config,
                         namespace=namespace,
                         quit_check_callback=quit_check_callback)

        self.es_context = self.config.elasticsearch.elasticsearch_class(
            config=self.config.elasticsearch)
        self.metrics = markus.get_metrics(namespace)
Esempio n. 8
0
 def __init__(self, config, quit_check_callback=None):
     self.config = config
     self._CreateError = boto.exception.StorageCreateError
     self.ResponseError = (
         boto.exception.StorageResponseError,
         KeyNotFound
     )
     self._bucket_cache = {}
     self.metrics = markus.get_metrics(config.boto_metrics_prefix)
Esempio n. 9
0
    def __init__(self, config):
        super(BetaVersionRule, self).__init__(config)
        self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE,
                                   default_ttl=self.SHORT_CACHE_TTL)
        self.metrics = markus.get_metrics('processor.betaversionrule')

        # NOTE(willkg): These config values come from Processor2015 instance and are
        # used for lookup in product_versions
        self.conn_context = config.database_class(config)
Esempio n. 10
0
    def test_print_records(self):
        # NOTE(willkg): .print_records() prints to stdout and is mostly used
        # for debugging tests. So we're just going to run it and make sure it
        # doesn't throw errors.
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics("foobar")
            mymetrics.incr("key1")

            mm.print_records()
Esempio n. 11
0
def test_histogram(metricsmock):
    metrics = get_metrics("thing")

    with metricsmock as mm:
        metrics.histogram("foo", value=4321)

    assert mm.get_records() == [
        MetricsRecord("histogram", "thing.foo", 4321, [])
    ]
Esempio n. 12
0
    def test_configure_doesnt_affect_override(self):
        with MetricsMock() as mm:
            markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
            mymetrics = markus.get_metrics("foobar")
            mymetrics.incr("key1", value=1)

            assert mm.has_record(fun_name="incr", stat="foobar.key1", value=1)

            assert not mm.has_record(fun_name="incr", stat="foobar.key1", value=5)
Esempio n. 13
0
    def test_clear_records(self):
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics("foobar")
            mymetrics.incr("key1", value=1, tags=["env:stage"])

            assert len(mm.get_records()) == 1

            mm.clear_records()

            assert len(mm.get_records()) == 0
Esempio n. 14
0
    def test_clear_records(self):
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics('foobar')
            mymetrics.incr('key1', value=1, tags=['env:stage'])

            assert len(mm.get_records()) == 1

            mm.clear_records()

            assert len(mm.get_records()) == 0
Esempio n. 15
0
def test_timer_decorator(metricsmock):
    metrics = get_metrics('thing')

    @metrics.timer_decorator('long_fun')
    def something():
        print('blah')

    with metricsmock as mm:
        something()

    assert mm.has_record(fun_name='timing', stat='thing.long_fun')
Esempio n. 16
0
def test_timer_decorator(metricsmock):
    metrics = get_metrics("thing")

    @metrics.timer_decorator("long_fun")
    def something():
        print("blah")

    with metricsmock as mm:
        something()

    assert mm.has_record(fun_name="timing", stat="thing.long_fun")
Esempio n. 17
0
    def test_has_record(self):
        # NOTE(willkg): .has_record() is implemented using .filter_records() so
        # we can test that aggressively and just make sure the .has_record()
        # wrapper works fine.
        #
        # If that ever changes, we should update this test.
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics("foobar")
            mymetrics.incr("key1", value=1)

            assert mm.has_record(fun_name="incr", stat="foobar.key1", value=1)

            assert not mm.has_record(fun_name="incr", stat="foobar.key1", value=5)
Esempio n. 18
0
    def __init__(self, dump_field, symbols_urls, command_pathname, command_line, kill_timeout,
                 symbol_tmp_path, symbol_cache_path, tmp_storage_path):
        super().__init__()
        self.dump_field = dump_field
        self.symbols_urls = symbols_urls
        self.command_pathname = command_pathname
        self.command_line = command_line
        self.kill_timeout = kill_timeout
        self.symbol_tmp_path = symbol_tmp_path
        self.symbol_cache_path = symbol_cache_path
        self.tmp_storage_path = tmp_storage_path

        self.metrics = markus.get_metrics('processor.breakpadstackwalkerrule')
Esempio n. 19
0
def get_metrics(namespace):
    global _configured
    if not _configured:
        STATSD_HOST = config('STATSD_HOST', 'localhost')
        STATSD_PORT = config('STATSD_PORT', default=8125)
        STATSD_NAMESPACE = config('STATSD_NAMESPACE', default='')

        FILE_METRICS_BASE_DIR = config('MARKUS_FILE_METRICS_BASE_DIR',
                                       default='/tmp')

        # For more options see
        # http://markus.readthedocs.io/en/latest/usage.html#markus-configure
        log_metrics_config = config('LOG_METRICS', default='datadog')
        if log_metrics_config == 'logging':
            markus.configure([{
                'class': 'markus.backends.logging.LoggingMetrics',
                'options': {
                    'logger_name': 'metrics'
                }
            }])
        elif log_metrics_config == 'cloudwatch':
            markus.configure([{
                'class':
                'markus.backends.cloudwatch.CloudwatchMetrics',
            }])
        elif log_metrics_config == 'datadog':
            markus.configure([{
                'class': 'markus.backends.datadog.DatadogMetrics',
                'options': {
                    'statsd_host': STATSD_HOST,
                    'statsd_port': STATSD_PORT,
                    'statsd_namespace': STATSD_NAMESPACE,
                }
            }])
        elif log_metrics_config == 'void':
            markus.configure([{
                'class': 'buildhub.configure_markus.VoidMetrics',
            }])
        elif log_metrics_config == 'file':
            markus.configure([{
                'class': 'buildhub.configure_markus.FileMetrics',
                'options': {
                    'base_dir': FILE_METRICS_BASE_DIR,
                }
            }])
        else:
            raise NotImplementedError(
                f'Unrecognized LOG_METRICS value {log_metrics_config}')
        _configured = True

    return markus.get_metrics(namespace)
Esempio n. 20
0
    def __init__(self, config, namespace='', quit_check_callback=None):
        super(ESCrashStorage,
              self).__init__(config,
                             namespace=namespace,
                             quit_check_callback=quit_check_callback)

        # Ok, it's sane, so let's continue.
        self.es_context = self.config.elasticsearch.elasticsearch_class(
            config=self.config.elasticsearch)

        self.transaction = config.transaction_executor_class(
            config, self.es_context, quit_check_callback)

        self.metrics = markus.get_metrics(namespace)
Esempio n. 21
0
    def test_filter_records_tags(self):
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics('foobar')
            mymetrics.incr('key1', value=1, tags=['env:stage'])
            mymetrics.incr('key2', value=3, tags=['env:prod'])

            key1_metrics = mm.filter_records(tags=['env:stage'], )
            assert len(key1_metrics) == 1
            assert key1_metrics[0][1] == 'foobar.key1'

            key1_metrics = mm.filter_records(tags=['env:prod'], )
            assert len(key1_metrics) == 1
            assert key1_metrics[0][1] == 'foobar.key2'

            key1_metrics = mm.filter_records(tags=['env:dev'], )
            assert len(key1_metrics) == 0
Esempio n. 22
0
    def test_filter_records_tags(self):
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics("foobar")
            mymetrics.incr("key1", value=1, tags=["env:stage"])
            mymetrics.incr("key2", value=3, tags=["env:prod"])

            key1_metrics = mm.filter_records(tags=["env:stage"])
            assert len(key1_metrics) == 1
            assert key1_metrics[0][1] == "foobar.key1"

            key1_metrics = mm.filter_records(tags=["env:prod"])
            assert len(key1_metrics) == 1
            assert key1_metrics[0][1] == "foobar.key2"

            key1_metrics = mm.filter_records(tags=["env:dev"])
            assert len(key1_metrics) == 0
Esempio n. 23
0
    def test_filter_records_value(self):
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics("foobar")
            mymetrics.incr("key1", value=1, tags=["env:stage"])

            key1_metrics = mm.filter_records(fun_name="incr", stat="foobar.key1")
            assert len(key1_metrics) == 1

            key1_metrics = mm.filter_records(
                fun_name="incr", stat="foobar.key1", value=1
            )
            assert len(key1_metrics) == 1

            key1_metrics = mm.filter_records(
                fun_name="incr", stat="foobar.key1", value=5
            )
            assert len(key1_metrics) == 0
Esempio n. 24
0
    def test_print_on_failure(self, capsys):
        with MetricsMock() as mm:
            markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
            mymetrics = markus.get_metrics("foobar")
            mymetrics.histogram("keymultiple", value=1)
            mymetrics.histogram("keymultiple", value=1)

            with pytest.raises(AssertionError):
                mm.assert_histogram_once(stat="foobar.keymultiple")

            # On assertion error, the assert_* methods will print the metrics
            # records to stdout.
            captured = capsys.readouterr()
            expected = (
                "<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n"
                "<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n"
            )
            assert captured.out == expected
Esempio n. 25
0
    def test_histogram_helpers(self):
        with MetricsMock() as mm:
            markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
            mymetrics = markus.get_metrics("foobar")
            mymetrics.histogram("key1", value=1)
            mymetrics.histogram("keymultiple", value=1)
            mymetrics.histogram("keymultiple", value=1)

            mm.assert_histogram(stat="foobar.key1")

            mm.assert_histogram_once(stat="foobar.key1")
            with pytest.raises(AssertionError):
                mm.assert_histogram_once(stat="foobar.keymultiple")

            mm.assert_not_histogram(stat="foobar.keynot")
            mm.assert_not_histogram(stat="foobar.key1", value=5)
            with pytest.raises(AssertionError):
                mm.assert_not_histogram(stat="foobar.key1")
Esempio n. 26
0
    def __init__(self, config, namespace='', quit_check_callback=None):
        super(ESCrashStorage, self).__init__(
            config,
            namespace=namespace,
            quit_check_callback=quit_check_callback
        )

        # Ok, it's sane, so let's continue.
        self.es_context = self.config.elasticsearch.elasticsearch_class(
            config=self.config.elasticsearch
        )

        self.transaction = config.transaction_executor_class(
            config,
            self.es_context,
            quit_check_callback
        )

        self.metrics = markus.get_metrics(namespace)
Esempio n. 27
0
    def test_configure_doesnt_affect_override(self):
        with MetricsMock() as mm:
            markus.configure([{
                'class': 'markus.backends.logging.LoggingMetrics'
            }])
            mymetrics = markus.get_metrics('foobar')
            mymetrics.incr('key1', value=1)

            assert mm.has_record(
                fun_name='incr',
                stat='foobar.key1',
                value=1,
            )

            assert not mm.has_record(
                fun_name='incr',
                stat='foobar.key1',
                value=5,
            )
Esempio n. 28
0
    def __init__(self,
                 hostname,
                 statsd_host=None,
                 statsd_port=None,
                 namespace="autopush",
                 flush_interval=10):
        markus.configure(backends=[{
            'class': 'markus.backends.datadog.DatadogMetrics',
            'options': {
                'statsd_host': statsd_host,
                'statsd_port': statsd_port,
            }
        }])
        self._client = markus.get_metrics(namespace)
        self._host = hostname
        self._namespace = namespace

        self._metrics = []
        self._flush_interval = flush_interval
        self._thread = None
        self._lock = threading.RLock()

        self.start()
Esempio n. 29
0
    def test_filter_records_value(self):
        with MetricsMock() as mm:
            mymetrics = markus.get_metrics('foobar')
            mymetrics.incr('key1', value=1, tags=['env:stage'])

            key1_metrics = mm.filter_records(
                fun_name='incr',
                stat='foobar.key1',
            )
            assert len(key1_metrics) == 1

            key1_metrics = mm.filter_records(
                fun_name='incr',
                stat='foobar.key1',
                value=1,
            )
            assert len(key1_metrics) == 1

            key1_metrics = mm.filter_records(
                fun_name='incr',
                stat='foobar.key1',
                value=5,
            )
            assert len(key1_metrics) == 0
Esempio n. 30
0
)
from crashstats.cron.models import Job, Log
from crashstats.cron.utils import (
    convert_frequency,
    convert_time,
    format_datetime,
    get_matching_job_specs,
    get_run_times,
    time_to_run,
)
from socorro.lib.sentry_client import capture_error


logger = logging.getLogger('crashstats.cron')

metrics = markus.get_metrics('cron')


class Command(BaseCommand):
    help = 'Run cron jobs.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--job',
            help='Run a specific job.'
        )
        parser.add_argument(
            '--force', action='store_true',
            help='Force a job to run even if it is not time.'
        )
        parser.add_argument(
Esempio n. 31
0
from antenna.throttler import (
    REJECT,
    FAKEACCEPT,
    RESULT_TO_TEXT,
    Throttler,
)
from antenna.util import (
    create_crash_id,
    sanitize_dump_name,
    utc_now,
    validate_crash_id,
)


logger = logging.getLogger(__name__)
mymetrics = markus.get_metrics('breakpad_resource')


#: Maximum number of attempts to save a crash before we give up
MAX_ATTEMPTS = 20

#: SAVE and PUBLISH states of the crash mover
STATE_SAVE = 'save'
STATE_PUBLISH = 'publish'


class CrashReport:
    """Crash report structure."""

    def __init__(self, raw_crash, dumps, crash_id, errors=0):
        self.raw_crash = raw_crash
Esempio n. 32
0
 def __init__(self, config, namespace='', quit_check_callback=None):
     self.config = config
     self.metrics = markus.get_metrics(self.config.metrics_prefix)
Esempio n. 33
0
from socorro.cron.base import convert_frequency
from socorro.lib import raven_client
from socorro.lib.datetimeutil import utc_now, timesince
from socorro.lib.dbutil import (
    SQLDidNotReturnSingleRow,
    SQLDidNotReturnSingleValue,
    execute_no_results,
    execute_query_fetchall,
    execute_query_iter,
    single_row_sql,
    single_value_sql,
)
from socorro.lib.transaction import transaction_context


metrics = markus.get_metrics('crontabber')


class JobNotFoundError(Exception):
    pass


class TimeDefinitionError(Exception):
    pass


class JobDescriptionError(Exception):
    pass


class BrokenJSONError(ValueError):
Esempio n. 34
0
File: diff.py Progetto: Pike/elmo
from difflib import SequenceMatcher

from django.shortcuts import render
from django import http
from django.views.generic.base import View

from life.models import Repository, Changeset

import hglib
import markus

from compare_locales.parser import getParser, FluentEntity
from compare_locales.compare.utils import AddRemove, Tree as DataTree


metrics = markus.get_metrics(__name__)


class BadRevision(Exception):
    "Revision could not be resolved"
    pass


class constdict(dict):
    '''Subclass dict to not allow modifications'''

    def __setitem__(self, key, value):
        raise NotImplementedError


class DiffView(View):
Esempio n. 35
0
            )
        except Http404 as e:
            # A 404 will be raised if the product doesn't exist, or if the
            # version does not exist for that product.
            # In the latter case, we want to redirect the user to that
            # product's home page. If the product is missing, superusers
            # should be sent to the admin panel to add that product, while
            # regular users will see a 404.
            if 'version' in str(e):
                return redirect(reverse('crashstats:product_home', args=(product,)))
            raise
        return view(request, *args, **kwargs)
    return inner


API_METRICS = markus.get_metrics('webapp.api')


def track_api_pageview(view):
    @functools.wraps(view)
    def inner(request, *args, **kwargs):
        response = view(request, *args, **kwargs)
        if response.status_code < 500:
            referer = request.META.get('HTTP_REFERER')
            if referer:
                # If the referer host is the same as the request host
                # that implies that the API was used as an AJAX request
                # in our main webapp itself. If so, don't track.
                referer_host = urlparse(referer).netloc
                if referer_host == request.META.get('HTTP_HOST'):
                    return response
Esempio n. 36
0
 def __init__(self, *args, **kwargs):
     super(BreakpadStackwalkerRule2015, self).__init__(*args, **kwargs)
     self.metrics = markus.get_metrics('processor.breakpadstackwalkerrule')
Esempio n. 37
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import

from buildbot.status.base import StatusReceiverMultiService, StatusReceiver
from twisted.python import log

import markus


metrics = markus.get_metrics('elmo-builds')


class MarkusStatusReceiver(StatusReceiverMultiService):
    '''StatusReceiver for markus metrics.
    '''

    def logPending(self):
        status = self.parent.getStatus()
        pending = current = 0
        for buildername in status.getBuilderNames():
            builder = status.getBuilder(buildername)
            pending += len(builder.getPendingBuilds())
            current += len(builder.getCurrentBuilds())
        metrics.gauge('pending_builds', pending)
        metrics.gauge('current_builds', current)

    def setServiceParent(self, parent):
        StatusReceiverMultiService.setServiceParent(self, parent)
Esempio n. 38
0
# Maximum number of retry attempts
MAX_ATTEMPTS = 5

# Number of seconds to wait for a response from server
CONNECTION_TIMEOUT = 60


class StdoutMetrics(BackendBase):
    def emit(self, record):
        click.echo("Elapsed time: %s %s %s" %
                   (record.stat_type, record.key, record.value))


markus.configure([{"class": StdoutMetrics}], raise_errors=True)
METRICS = markus.get_metrics()


@click.command()
@click.option(
    "--base-url",
    default="https://symbols.mozilla.org/",
    help="Base url to use for uploading SYM files.",
)
@click.option(
    "--auth-token",
    required=True,
    help="Auth token for uploading SYM files.",
)
@click.argument("symbolsfile")
@click.pass_context
Esempio n. 39
0
from django.contrib.auth import get_user_model
from mozilla_nimbus_shared import get_data

from experimenter.celery import app
from experimenter.experiments.api.v4.serializers import ExperimentRapidRecipeSerializer
from experimenter.experiments.changelog_utils import update_experiment_with_change_log
from experimenter.experiments.models import (
    Experiment,
    ExperimentBucketNamespace,
    ExperimentBucketRange,
    ExperimentChangeLog,
)
from experimenter.kinto import client

logger = get_task_logger(__name__)
metrics = markus.get_metrics("kinto.tasks")

NIMBUS_DATA = get_data()


@app.task
@metrics.timer_decorator("push_experiment_to_kinto.timing")
def push_experiment_to_kinto(experiment_id):
    metrics.incr("push_experiment_to_kinto.started")

    experiment = Experiment.objects.get(id=experiment_id)
    if not ExperimentBucketRange.objects.filter(experiment=experiment).exists():
        ExperimentBucketNamespace.request_namespace_buckets(
            experiment.recipe_slug,
            experiment,
            NIMBUS_DATA["ExperimentDesignPresets"]["empty_aa"]["preset"]["arguments"][
Esempio n. 40
0
from django.contrib.auth import get_user_model
from django.db import IntegrityError, transaction
from django.conf import settings
from celery.utils.log import get_task_logger

from experimenter import bugzilla
from experimenter import normandy
from experimenter.celery import app
from experimenter.experiments import email
from experimenter.experiments.constants import ExperimentConstants
from experimenter.experiments.changelog_utils import update_experiment_with_change_log
from experimenter.experiments.models import Experiment, ExperimentEmail
from experimenter.notifications.models import Notification

logger = get_task_logger(__name__)
metrics = markus.get_metrics("experiments.tasks")

NOTIFICATION_MESSAGE_CREATE_BUG = (
    'A <a target="_blank" rel="noreferrer noopener" href="{bug_url}">Bugzilla '
    "Ticket</a> was created for your experiment")
NOTIFICATION_MESSAGE_CREATE_BUG_FAILED = (
    "Experimenter failed to create a Bugzilla Ticket "
    "for your experiment.  Please contact an Experimenter "
    "Administrator on #ask-experimenter on Slack.")
NOTIFICATION_MESSAGE_UPDATE_BUG = (
    'The <a target="_blank" rel="noreferrer noopener" href="{bug_url}">'
    "Ticket</a> was updated with the details "
    "of this experiment")
NOTIFICATION_MESSAGE_UPDATE_BUG_FAILED = (
    "Experimenter failed to update the Bugzilla Ticket "
    "for your experiment.  Please contact an Experimenter "
Esempio n. 41
0
from configman import Namespace, class_converter
import markus
from psycopg2 import IntegrityError

from socorro.cron.base import BaseCronApp
from socorro.cron.mixins import as_backfill_cron_app
from socorro.lib.dbutil import execute_no_results, execute_query_fetchall
from socorro.lib.transaction import transaction_context


RAW_CRASH_PREFIX_TEMPLATE = 'v2/raw_crash/%s/%s/'
PROCESSED_CRASH_TEMPLATE = 'v1/processed_crash/%s'


metrics = markus.get_metrics('crontabber.verifyprocessed')


def check_crashids(entropy, boto_conn, bucket_name, date):
    """Checks crash ids for a given entropy and date."""
    bucket = boto_conn.get_bucket(bucket_name)

    raw_crash_key_prefix = RAW_CRASH_PREFIX_TEMPLATE % (entropy, date)

    missing = []
    for key_instance in bucket.list(raw_crash_key_prefix):
        raw_crash_key = key_instance.key
        crash_id = raw_crash_key.split('/')[-1]

        processed_crash_key = bucket.get_key(PROCESSED_CRASH_TEMPLATE % crash_id)
        if processed_crash_key is None:
Esempio n. 42
0
 def __init__(self, *args, **kwargs):
     super().__init__(*args, **kwargs)
     self.metrics = markus.get_metrics('processor.breakpadstackwalkerrule')
Esempio n. 43
0
from urllib3.util import Retry


from .consumer import ConsumerThread
from ..config import IXP_NETWORKS_UPDATE_INTERVAL
from ..structures import IPDBInfo, IXPNetwork
from ..ip_info_db import IPInfo_Prefix
from ..metrics import log_execution_time, get_tags


PEERINGDB_API_IXPFX = "https://www.peeringdb.com/api/ixpfx"
PEERINGDB_API_IXLAN = "https://www.peeringdb.com/api/ixlan"
PEERINGDB_API_IX = "https://www.peeringdb.com/api/ix"

LOGGER = logging.getLogger(__name__)
METRICS = markus.get_metrics(__name__)

thread = None


class PeeringDB:

    def __init__(self):
        retry_strategy = Retry(
            total=3,
            status_forcelist=[429, 500, 502, 503, 504],
            allowed_methods=["HEAD", "GET", "PUT",
                             "DELETE", "OPTIONS", "TRACE"],
            backoff_factor=3
        )
        adapter = HTTPAdapter(max_retries=retry_strategy)
Esempio n. 44
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import logging

from configman import RequiredConfig
import markus


metrics = markus.get_metrics('processor.rule')


class Rule(RequiredConfig):
    """Base class for transform rules

    Provides structure for calling rules during the processor pipeline and also
    has some useful utilities for rules.

    """

    def __init__(self, config=None, quit_check_callback=None):
        self.config = config
        self.quit_check_callback = quit_check_callback
        self.logger = logging.getLogger(__name__ + '.' + self.__class__.__name__)

    def predicate(self, raw_crash, raw_dumps, processed_crash, processor_meta_data):
        """Determines whether to run the action for this crash

        :arg raw_crash: the raw crash data
        :arg raw_dumps: any minidumps associated with this crash
Esempio n. 45
0
from email.mime.application import MIMEApplication
from email.utils import parseaddr
import json
import os

from botocore.exceptions import ClientError
import markus
import logging

from django.apps import apps
from django.conf import settings
from django.http import HttpResponse
from django.template.defaultfilters import linebreaksbr, urlize

logger = logging.getLogger('events')
metrics = markus.get_metrics('fx-private-relay')


def time_if_enabled(name):
    def timing_decorator(func):
        def func_wrapper(*args, **kwargs):
            ctx_manager = (metrics.timer(name) if settings.STATSD_ENABLED else
                           contextlib.nullcontext())
            with ctx_manager:
                return func(*args, **kwargs)

        return func_wrapper

    return timing_decorator

Esempio n. 46
0
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from collections import OrderedDict
import json
import logging

from everett.component import RequiredConfigMixin
import falcon
import markus

from antenna.util import get_version_info


logger = logging.getLogger(__name__)
mymetrics = markus.get_metrics('health')


class BrokenResource(RequiredConfigMixin):
    """Handle ``/__broken__`` endpoint."""

    def __init__(self, config):
        self.config = config

    def on_get(self, req, resp):
        """Implement GET HTTP request."""
        mymetrics.incr('broken.count')
        # This is intentional breakage
        raise Exception('intentional exception')