def __init__(self, config, namespace='', quit_check_callback=None): super().__init__(config, namespace=namespace, quit_check_callback=quit_check_callback) self.es_context = self.config.elasticsearch.elasticsearch_class( config=self.config.elasticsearch ) self.metrics = markus.get_metrics(namespace)
def setup_metrics(config, local_unused, args_unused): """Sets up markus and adds a metrics client to config :returns: Markus MetricsInterface """ backends = [] for backend in config.metricscfg.markus_backends: if backend == 'markus.backends.statsd.StatsdMetrics': backends.append({ 'class': 'markus.backends.statsd.StatsdMetrics', 'options': { 'statsd_host': config.metricscfg.statsd_host, 'statsd_port': config.metricscfg.statsd_port, } }) elif backend == 'markus.backends.datadog.DatadogMetrics': backends.append({ 'class': 'markus.backends.datadog.DatadogMetrics', 'options': { 'statsd_host': config.metricscfg.statsd_host, 'statsd_port': config.metricscfg.statsd_port, } }) elif backend == 'markus.backends.logging.LoggingMetrics': backends.append({ 'class': 'markus.backends.logging.LoggingMetrics', }) else: raise ValueError('Invalid markus backend "%s"' % backend) markus.configure(backends=backends) return markus.get_metrics('')
def __init__(self, config): super().__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # For looking up version strings self.version_string_api = config.version_string_api self.session = session_with_retries()
def __init__(self, config, quit_check_callback=None): self.config = config self._CreateError = boto.exception.StorageCreateError self.ResponseError = (boto.exception.StorageResponseError, KeyNotFound) self._bucket_cache = {} self.metrics = markus.get_metrics(config.boto_metrics_prefix) self._connect_to_endpoint = boto.connect_s3 self._calling_format = config.calling_format
def test_timer_contextmanager(metricsmock): metrics = get_metrics("thing") with metricsmock as mm: with metrics.timer("long_fun"): print("blah") assert mm.has_record(fun_name="timing", stat="thing.long_fun")
def __init__(self, config, namespace="", quit_check_callback=None): super().__init__(config, namespace=namespace, quit_check_callback=quit_check_callback) self.es_context = self.config.elasticsearch.elasticsearch_class( config=self.config.elasticsearch) self.metrics = markus.get_metrics(namespace)
def __init__(self, config, quit_check_callback=None): self.config = config self._CreateError = boto.exception.StorageCreateError self.ResponseError = ( boto.exception.StorageResponseError, KeyNotFound ) self._bucket_cache = {} self.metrics = markus.get_metrics(config.boto_metrics_prefix)
def __init__(self, config): super(BetaVersionRule, self).__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # NOTE(willkg): These config values come from Processor2015 instance and are # used for lookup in product_versions self.conn_context = config.database_class(config)
def test_print_records(self): # NOTE(willkg): .print_records() prints to stdout and is mostly used # for debugging tests. So we're just going to run it and make sure it # doesn't throw errors. with MetricsMock() as mm: mymetrics = markus.get_metrics("foobar") mymetrics.incr("key1") mm.print_records()
def test_histogram(metricsmock): metrics = get_metrics("thing") with metricsmock as mm: metrics.histogram("foo", value=4321) assert mm.get_records() == [ MetricsRecord("histogram", "thing.foo", 4321, []) ]
def test_configure_doesnt_affect_override(self): with MetricsMock() as mm: markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}]) mymetrics = markus.get_metrics("foobar") mymetrics.incr("key1", value=1) assert mm.has_record(fun_name="incr", stat="foobar.key1", value=1) assert not mm.has_record(fun_name="incr", stat="foobar.key1", value=5)
def test_clear_records(self): with MetricsMock() as mm: mymetrics = markus.get_metrics("foobar") mymetrics.incr("key1", value=1, tags=["env:stage"]) assert len(mm.get_records()) == 1 mm.clear_records() assert len(mm.get_records()) == 0
def test_clear_records(self): with MetricsMock() as mm: mymetrics = markus.get_metrics('foobar') mymetrics.incr('key1', value=1, tags=['env:stage']) assert len(mm.get_records()) == 1 mm.clear_records() assert len(mm.get_records()) == 0
def test_timer_decorator(metricsmock): metrics = get_metrics('thing') @metrics.timer_decorator('long_fun') def something(): print('blah') with metricsmock as mm: something() assert mm.has_record(fun_name='timing', stat='thing.long_fun')
def test_timer_decorator(metricsmock): metrics = get_metrics("thing") @metrics.timer_decorator("long_fun") def something(): print("blah") with metricsmock as mm: something() assert mm.has_record(fun_name="timing", stat="thing.long_fun")
def test_has_record(self): # NOTE(willkg): .has_record() is implemented using .filter_records() so # we can test that aggressively and just make sure the .has_record() # wrapper works fine. # # If that ever changes, we should update this test. with MetricsMock() as mm: mymetrics = markus.get_metrics("foobar") mymetrics.incr("key1", value=1) assert mm.has_record(fun_name="incr", stat="foobar.key1", value=1) assert not mm.has_record(fun_name="incr", stat="foobar.key1", value=5)
def __init__(self, dump_field, symbols_urls, command_pathname, command_line, kill_timeout, symbol_tmp_path, symbol_cache_path, tmp_storage_path): super().__init__() self.dump_field = dump_field self.symbols_urls = symbols_urls self.command_pathname = command_pathname self.command_line = command_line self.kill_timeout = kill_timeout self.symbol_tmp_path = symbol_tmp_path self.symbol_cache_path = symbol_cache_path self.tmp_storage_path = tmp_storage_path self.metrics = markus.get_metrics('processor.breakpadstackwalkerrule')
def get_metrics(namespace): global _configured if not _configured: STATSD_HOST = config('STATSD_HOST', 'localhost') STATSD_PORT = config('STATSD_PORT', default=8125) STATSD_NAMESPACE = config('STATSD_NAMESPACE', default='') FILE_METRICS_BASE_DIR = config('MARKUS_FILE_METRICS_BASE_DIR', default='/tmp') # For more options see # http://markus.readthedocs.io/en/latest/usage.html#markus-configure log_metrics_config = config('LOG_METRICS', default='datadog') if log_metrics_config == 'logging': markus.configure([{ 'class': 'markus.backends.logging.LoggingMetrics', 'options': { 'logger_name': 'metrics' } }]) elif log_metrics_config == 'cloudwatch': markus.configure([{ 'class': 'markus.backends.cloudwatch.CloudwatchMetrics', }]) elif log_metrics_config == 'datadog': markus.configure([{ 'class': 'markus.backends.datadog.DatadogMetrics', 'options': { 'statsd_host': STATSD_HOST, 'statsd_port': STATSD_PORT, 'statsd_namespace': STATSD_NAMESPACE, } }]) elif log_metrics_config == 'void': markus.configure([{ 'class': 'buildhub.configure_markus.VoidMetrics', }]) elif log_metrics_config == 'file': markus.configure([{ 'class': 'buildhub.configure_markus.FileMetrics', 'options': { 'base_dir': FILE_METRICS_BASE_DIR, } }]) else: raise NotImplementedError( f'Unrecognized LOG_METRICS value {log_metrics_config}') _configured = True return markus.get_metrics(namespace)
def __init__(self, config, namespace='', quit_check_callback=None): super(ESCrashStorage, self).__init__(config, namespace=namespace, quit_check_callback=quit_check_callback) # Ok, it's sane, so let's continue. self.es_context = self.config.elasticsearch.elasticsearch_class( config=self.config.elasticsearch) self.transaction = config.transaction_executor_class( config, self.es_context, quit_check_callback) self.metrics = markus.get_metrics(namespace)
def test_filter_records_tags(self): with MetricsMock() as mm: mymetrics = markus.get_metrics('foobar') mymetrics.incr('key1', value=1, tags=['env:stage']) mymetrics.incr('key2', value=3, tags=['env:prod']) key1_metrics = mm.filter_records(tags=['env:stage'], ) assert len(key1_metrics) == 1 assert key1_metrics[0][1] == 'foobar.key1' key1_metrics = mm.filter_records(tags=['env:prod'], ) assert len(key1_metrics) == 1 assert key1_metrics[0][1] == 'foobar.key2' key1_metrics = mm.filter_records(tags=['env:dev'], ) assert len(key1_metrics) == 0
def test_filter_records_tags(self): with MetricsMock() as mm: mymetrics = markus.get_metrics("foobar") mymetrics.incr("key1", value=1, tags=["env:stage"]) mymetrics.incr("key2", value=3, tags=["env:prod"]) key1_metrics = mm.filter_records(tags=["env:stage"]) assert len(key1_metrics) == 1 assert key1_metrics[0][1] == "foobar.key1" key1_metrics = mm.filter_records(tags=["env:prod"]) assert len(key1_metrics) == 1 assert key1_metrics[0][1] == "foobar.key2" key1_metrics = mm.filter_records(tags=["env:dev"]) assert len(key1_metrics) == 0
def test_filter_records_value(self): with MetricsMock() as mm: mymetrics = markus.get_metrics("foobar") mymetrics.incr("key1", value=1, tags=["env:stage"]) key1_metrics = mm.filter_records(fun_name="incr", stat="foobar.key1") assert len(key1_metrics) == 1 key1_metrics = mm.filter_records( fun_name="incr", stat="foobar.key1", value=1 ) assert len(key1_metrics) == 1 key1_metrics = mm.filter_records( fun_name="incr", stat="foobar.key1", value=5 ) assert len(key1_metrics) == 0
def test_print_on_failure(self, capsys): with MetricsMock() as mm: markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}]) mymetrics = markus.get_metrics("foobar") mymetrics.histogram("keymultiple", value=1) mymetrics.histogram("keymultiple", value=1) with pytest.raises(AssertionError): mm.assert_histogram_once(stat="foobar.keymultiple") # On assertion error, the assert_* methods will print the metrics # records to stdout. captured = capsys.readouterr() expected = ( "<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n" "<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n" ) assert captured.out == expected
def test_histogram_helpers(self): with MetricsMock() as mm: markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}]) mymetrics = markus.get_metrics("foobar") mymetrics.histogram("key1", value=1) mymetrics.histogram("keymultiple", value=1) mymetrics.histogram("keymultiple", value=1) mm.assert_histogram(stat="foobar.key1") mm.assert_histogram_once(stat="foobar.key1") with pytest.raises(AssertionError): mm.assert_histogram_once(stat="foobar.keymultiple") mm.assert_not_histogram(stat="foobar.keynot") mm.assert_not_histogram(stat="foobar.key1", value=5) with pytest.raises(AssertionError): mm.assert_not_histogram(stat="foobar.key1")
def __init__(self, config, namespace='', quit_check_callback=None): super(ESCrashStorage, self).__init__( config, namespace=namespace, quit_check_callback=quit_check_callback ) # Ok, it's sane, so let's continue. self.es_context = self.config.elasticsearch.elasticsearch_class( config=self.config.elasticsearch ) self.transaction = config.transaction_executor_class( config, self.es_context, quit_check_callback ) self.metrics = markus.get_metrics(namespace)
def test_configure_doesnt_affect_override(self): with MetricsMock() as mm: markus.configure([{ 'class': 'markus.backends.logging.LoggingMetrics' }]) mymetrics = markus.get_metrics('foobar') mymetrics.incr('key1', value=1) assert mm.has_record( fun_name='incr', stat='foobar.key1', value=1, ) assert not mm.has_record( fun_name='incr', stat='foobar.key1', value=5, )
def __init__(self, hostname, statsd_host=None, statsd_port=None, namespace="autopush", flush_interval=10): markus.configure(backends=[{ 'class': 'markus.backends.datadog.DatadogMetrics', 'options': { 'statsd_host': statsd_host, 'statsd_port': statsd_port, } }]) self._client = markus.get_metrics(namespace) self._host = hostname self._namespace = namespace self._metrics = [] self._flush_interval = flush_interval self._thread = None self._lock = threading.RLock() self.start()
def test_filter_records_value(self): with MetricsMock() as mm: mymetrics = markus.get_metrics('foobar') mymetrics.incr('key1', value=1, tags=['env:stage']) key1_metrics = mm.filter_records( fun_name='incr', stat='foobar.key1', ) assert len(key1_metrics) == 1 key1_metrics = mm.filter_records( fun_name='incr', stat='foobar.key1', value=1, ) assert len(key1_metrics) == 1 key1_metrics = mm.filter_records( fun_name='incr', stat='foobar.key1', value=5, ) assert len(key1_metrics) == 0
) from crashstats.cron.models import Job, Log from crashstats.cron.utils import ( convert_frequency, convert_time, format_datetime, get_matching_job_specs, get_run_times, time_to_run, ) from socorro.lib.sentry_client import capture_error logger = logging.getLogger('crashstats.cron') metrics = markus.get_metrics('cron') class Command(BaseCommand): help = 'Run cron jobs.' def add_arguments(self, parser): parser.add_argument( '--job', help='Run a specific job.' ) parser.add_argument( '--force', action='store_true', help='Force a job to run even if it is not time.' ) parser.add_argument(
from antenna.throttler import ( REJECT, FAKEACCEPT, RESULT_TO_TEXT, Throttler, ) from antenna.util import ( create_crash_id, sanitize_dump_name, utc_now, validate_crash_id, ) logger = logging.getLogger(__name__) mymetrics = markus.get_metrics('breakpad_resource') #: Maximum number of attempts to save a crash before we give up MAX_ATTEMPTS = 20 #: SAVE and PUBLISH states of the crash mover STATE_SAVE = 'save' STATE_PUBLISH = 'publish' class CrashReport: """Crash report structure.""" def __init__(self, raw_crash, dumps, crash_id, errors=0): self.raw_crash = raw_crash
def __init__(self, config, namespace='', quit_check_callback=None): self.config = config self.metrics = markus.get_metrics(self.config.metrics_prefix)
from socorro.cron.base import convert_frequency from socorro.lib import raven_client from socorro.lib.datetimeutil import utc_now, timesince from socorro.lib.dbutil import ( SQLDidNotReturnSingleRow, SQLDidNotReturnSingleValue, execute_no_results, execute_query_fetchall, execute_query_iter, single_row_sql, single_value_sql, ) from socorro.lib.transaction import transaction_context metrics = markus.get_metrics('crontabber') class JobNotFoundError(Exception): pass class TimeDefinitionError(Exception): pass class JobDescriptionError(Exception): pass class BrokenJSONError(ValueError):
from difflib import SequenceMatcher from django.shortcuts import render from django import http from django.views.generic.base import View from life.models import Repository, Changeset import hglib import markus from compare_locales.parser import getParser, FluentEntity from compare_locales.compare.utils import AddRemove, Tree as DataTree metrics = markus.get_metrics(__name__) class BadRevision(Exception): "Revision could not be resolved" pass class constdict(dict): '''Subclass dict to not allow modifications''' def __setitem__(self, key, value): raise NotImplementedError class DiffView(View):
) except Http404 as e: # A 404 will be raised if the product doesn't exist, or if the # version does not exist for that product. # In the latter case, we want to redirect the user to that # product's home page. If the product is missing, superusers # should be sent to the admin panel to add that product, while # regular users will see a 404. if 'version' in str(e): return redirect(reverse('crashstats:product_home', args=(product,))) raise return view(request, *args, **kwargs) return inner API_METRICS = markus.get_metrics('webapp.api') def track_api_pageview(view): @functools.wraps(view) def inner(request, *args, **kwargs): response = view(request, *args, **kwargs) if response.status_code < 500: referer = request.META.get('HTTP_REFERER') if referer: # If the referer host is the same as the request host # that implies that the API was used as an AJAX request # in our main webapp itself. If so, don't track. referer_host = urlparse(referer).netloc if referer_host == request.META.get('HTTP_HOST'): return response
def __init__(self, *args, **kwargs): super(BreakpadStackwalkerRule2015, self).__init__(*args, **kwargs) self.metrics = markus.get_metrics('processor.breakpadstackwalkerrule')
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import from buildbot.status.base import StatusReceiverMultiService, StatusReceiver from twisted.python import log import markus metrics = markus.get_metrics('elmo-builds') class MarkusStatusReceiver(StatusReceiverMultiService): '''StatusReceiver for markus metrics. ''' def logPending(self): status = self.parent.getStatus() pending = current = 0 for buildername in status.getBuilderNames(): builder = status.getBuilder(buildername) pending += len(builder.getPendingBuilds()) current += len(builder.getCurrentBuilds()) metrics.gauge('pending_builds', pending) metrics.gauge('current_builds', current) def setServiceParent(self, parent): StatusReceiverMultiService.setServiceParent(self, parent)
# Maximum number of retry attempts MAX_ATTEMPTS = 5 # Number of seconds to wait for a response from server CONNECTION_TIMEOUT = 60 class StdoutMetrics(BackendBase): def emit(self, record): click.echo("Elapsed time: %s %s %s" % (record.stat_type, record.key, record.value)) markus.configure([{"class": StdoutMetrics}], raise_errors=True) METRICS = markus.get_metrics() @click.command() @click.option( "--base-url", default="https://symbols.mozilla.org/", help="Base url to use for uploading SYM files.", ) @click.option( "--auth-token", required=True, help="Auth token for uploading SYM files.", ) @click.argument("symbolsfile") @click.pass_context
from django.contrib.auth import get_user_model from mozilla_nimbus_shared import get_data from experimenter.celery import app from experimenter.experiments.api.v4.serializers import ExperimentRapidRecipeSerializer from experimenter.experiments.changelog_utils import update_experiment_with_change_log from experimenter.experiments.models import ( Experiment, ExperimentBucketNamespace, ExperimentBucketRange, ExperimentChangeLog, ) from experimenter.kinto import client logger = get_task_logger(__name__) metrics = markus.get_metrics("kinto.tasks") NIMBUS_DATA = get_data() @app.task @metrics.timer_decorator("push_experiment_to_kinto.timing") def push_experiment_to_kinto(experiment_id): metrics.incr("push_experiment_to_kinto.started") experiment = Experiment.objects.get(id=experiment_id) if not ExperimentBucketRange.objects.filter(experiment=experiment).exists(): ExperimentBucketNamespace.request_namespace_buckets( experiment.recipe_slug, experiment, NIMBUS_DATA["ExperimentDesignPresets"]["empty_aa"]["preset"]["arguments"][
from django.contrib.auth import get_user_model from django.db import IntegrityError, transaction from django.conf import settings from celery.utils.log import get_task_logger from experimenter import bugzilla from experimenter import normandy from experimenter.celery import app from experimenter.experiments import email from experimenter.experiments.constants import ExperimentConstants from experimenter.experiments.changelog_utils import update_experiment_with_change_log from experimenter.experiments.models import Experiment, ExperimentEmail from experimenter.notifications.models import Notification logger = get_task_logger(__name__) metrics = markus.get_metrics("experiments.tasks") NOTIFICATION_MESSAGE_CREATE_BUG = ( 'A <a target="_blank" rel="noreferrer noopener" href="{bug_url}">Bugzilla ' "Ticket</a> was created for your experiment") NOTIFICATION_MESSAGE_CREATE_BUG_FAILED = ( "Experimenter failed to create a Bugzilla Ticket " "for your experiment. Please contact an Experimenter " "Administrator on #ask-experimenter on Slack.") NOTIFICATION_MESSAGE_UPDATE_BUG = ( 'The <a target="_blank" rel="noreferrer noopener" href="{bug_url}">' "Ticket</a> was updated with the details " "of this experiment") NOTIFICATION_MESSAGE_UPDATE_BUG_FAILED = ( "Experimenter failed to update the Bugzilla Ticket " "for your experiment. Please contact an Experimenter "
from configman import Namespace, class_converter import markus from psycopg2 import IntegrityError from socorro.cron.base import BaseCronApp from socorro.cron.mixins import as_backfill_cron_app from socorro.lib.dbutil import execute_no_results, execute_query_fetchall from socorro.lib.transaction import transaction_context RAW_CRASH_PREFIX_TEMPLATE = 'v2/raw_crash/%s/%s/' PROCESSED_CRASH_TEMPLATE = 'v1/processed_crash/%s' metrics = markus.get_metrics('crontabber.verifyprocessed') def check_crashids(entropy, boto_conn, bucket_name, date): """Checks crash ids for a given entropy and date.""" bucket = boto_conn.get_bucket(bucket_name) raw_crash_key_prefix = RAW_CRASH_PREFIX_TEMPLATE % (entropy, date) missing = [] for key_instance in bucket.list(raw_crash_key_prefix): raw_crash_key = key_instance.key crash_id = raw_crash_key.split('/')[-1] processed_crash_key = bucket.get_key(PROCESSED_CRASH_TEMPLATE % crash_id) if processed_crash_key is None:
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.metrics = markus.get_metrics('processor.breakpadstackwalkerrule')
from urllib3.util import Retry from .consumer import ConsumerThread from ..config import IXP_NETWORKS_UPDATE_INTERVAL from ..structures import IPDBInfo, IXPNetwork from ..ip_info_db import IPInfo_Prefix from ..metrics import log_execution_time, get_tags PEERINGDB_API_IXPFX = "https://www.peeringdb.com/api/ixpfx" PEERINGDB_API_IXLAN = "https://www.peeringdb.com/api/ixlan" PEERINGDB_API_IX = "https://www.peeringdb.com/api/ix" LOGGER = logging.getLogger(__name__) METRICS = markus.get_metrics(__name__) thread = None class PeeringDB: def __init__(self): retry_strategy = Retry( total=3, status_forcelist=[429, 500, 502, 503, 504], allowed_methods=["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"], backoff_factor=3 ) adapter = HTTPAdapter(max_retries=retry_strategy)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import logging from configman import RequiredConfig import markus metrics = markus.get_metrics('processor.rule') class Rule(RequiredConfig): """Base class for transform rules Provides structure for calling rules during the processor pipeline and also has some useful utilities for rules. """ def __init__(self, config=None, quit_check_callback=None): self.config = config self.quit_check_callback = quit_check_callback self.logger = logging.getLogger(__name__ + '.' + self.__class__.__name__) def predicate(self, raw_crash, raw_dumps, processed_crash, processor_meta_data): """Determines whether to run the action for this crash :arg raw_crash: the raw crash data :arg raw_dumps: any minidumps associated with this crash
from email.mime.application import MIMEApplication from email.utils import parseaddr import json import os from botocore.exceptions import ClientError import markus import logging from django.apps import apps from django.conf import settings from django.http import HttpResponse from django.template.defaultfilters import linebreaksbr, urlize logger = logging.getLogger('events') metrics = markus.get_metrics('fx-private-relay') def time_if_enabled(name): def timing_decorator(func): def func_wrapper(*args, **kwargs): ctx_manager = (metrics.timer(name) if settings.STATSD_ENABLED else contextlib.nullcontext()) with ctx_manager: return func(*args, **kwargs) return func_wrapper return timing_decorator
# License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from collections import OrderedDict import json import logging from everett.component import RequiredConfigMixin import falcon import markus from antenna.util import get_version_info logger = logging.getLogger(__name__) mymetrics = markus.get_metrics('health') class BrokenResource(RequiredConfigMixin): """Handle ``/__broken__`` endpoint.""" def __init__(self, config): self.config = config def on_get(self, req, resp): """Implement GET HTTP request.""" mymetrics.incr('broken.count') # This is intentional breakage raise Exception('intentional exception')