def testMetricsUploadTimer(self):
"""Test the periodic metric upload system."""
def _OnQueryMetric(min_metrics, max_metrics, metrics):
self.assertTrue(len(metrics) >= min_metrics and len(metrics) <= max_metrics,
'%d not in [%d-%d]' % (len(metrics), min_metrics, max_metrics))
for m in metrics:
self.assertTrue(m.timestamp % 3 == 0)
payload = DotDict(json.loads(metrics[0].payload))
keys = counters.counters.flatten().keys()
for k in keys:
self.assertTrue(k in payload, 'Payload did not contain record for counter %s' % k)
self.stop()
start_time = time.time()
cluster_name = 'test_group_key'
interval = MetricInterval('test_interval', 3)
group_key = Metric.EncodeGroupKey(cluster_name, interval)
rate = counters.define_rate('metricstest.rate', 'Test rate')
Metric.StartMetricUpload(self._client, cluster_name, interval)
IOLoop.current().add_timeout(start_time + 7, self.stop)
rate.increment()
self.wait(timeout=10)
end_time = time.time()
Metric.StopMetricUpload(group_key)
Metric.QueryTimespan(self._client, group_key, start_time, end_time, partial(_OnQueryMetric, 2, 3))
Metric.QueryTimespan(self._client, group_key, start_time + 4, end_time, partial(_OnQueryMetric, 1, 2))
Metric.QueryTimespan(self._client, group_key, start_time, end_time - 4, partial(_OnQueryMetric, 1, 2))
Metric.QueryTimespan(self._client, group_key, None, end_time, partial(_OnQueryMetric, 2, 3))
Metric.QueryTimespan(self._client, group_key, start_time, None, partial(_OnQueryMetric, 2, 3))
# Setting both start_time and end_time to None fails.
self.assertRaises(AssertionError,
Metric.QueryTimespan, self._client, group_key, None, None, partial(_OnQueryMetric, 2, 3))
def __init__(self, table_name, read_write, name, ups):
"""'ups' is measured either as read or write capacity units per second.
"""
self._name = name
self._ups = ups
self._queue = []
self._last_rate_adjust = time.time()
self._unavailable_rate = 0.0
self._need_adj = False
qps_counter = backoff_counter = None
if table_name in kSaveMetricsFor:
rw_str = 'write' if read_write else 'read'
qps_counter = counters.define_rate('viewfinder.dynamodb.qps.%s_%s' % (table_name, rw_str),
'Dynamodb %s QPS on %s' % (rw_str, table_name), 1)
backoff_counter = counters.define_rate('viewfinder.dynamodb.backoff_per_sec.%s_%s' % (table_name, rw_str),
'Dynamodb %s backoff seconds per second on %s' % (rw_str, table_name), 1)
self._ups_rate = rate_limiter.RateLimiter(ups, qps_counter=qps_counter, backoff_counter=backoff_counter)
self._timeout = None
__author__ = '[email protected] (Peter Mattis)' import boto import time import urllib from boto.s3.connection import S3Connection from functools import partial from viewfinder.backend.base import constants, counters, util from viewfinder.backend.base.secrets import GetSecret from viewfinder.backend.storage.object_store import ObjectStore from viewfinder.backend.storage.async_s3 import AsyncS3Connection from xml.etree import ElementTree _puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min', 'Average S3 puts per minute.', 60) _secs_per_put = counters.define_average('viewfinder.s3.secs_per_put', 'Average time in seconds to complete each S3 put') _gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min', 'Average S3 gets per minute.', 60) class S3ObjectStore(ObjectStore): """Simple object storage interface supporting key/value pairs backed by S3. Methods that require network round-trips are asynchronous; they take a callback argument that will be invoked if the operation completes successfully. If the operation fails, then an exception is raised. To handle this exception, use a Barrier instance with the exception handler defined. """ def __init__(self, bucket_name, temporary=False, read_only=False): assert not temporary, 'temporary can only be specified True for file object store' self._bucket_name = bucket_name self._read_only = read_only
kMaxCapacityThrottleFraction = 0.75
# What fraction of the throughput capacity do we lose every time we receive a throttle from dynamodb.
# eg: 0.05 means that we'll lose 5% of our capacity every time.
kPerThrottleLostCapacityFraction = 0.1
# Minimum amount of time between rate adjustments, in seconds.
kMinRateAdjustmentPeriod = 1.0
DynDBRequest = namedtuple('DynDBRequest', ['method', 'request', 'op', 'execute_cb', 'finish_cb'])
_requests_queued = counters.define_total('viewfinder.dynamodb.requests_queued',
'Number of DynamoDB requests currently queued.')
# TODO: we should have this per table. A global counter is mostly meaningless.
_throttles_per_min = counters.define_rate('viewfinder.dynamodb.throttles_per_min',
'Number of throttling errors received from DynamoDB per minute.', 60)
# In addition to these counters, each RequestQueue may setup an extra two (one for QPS, one for backoff).
class RequestQueue(object):
"""Manages the complexity of tracking successes and failures and
estimating backoff delays for a request queue.
"""
def __init__(self, table_name, read_write, name, ups):
"""'ups' is measured either as read or write capacity units per second.
"""
self._name = name
self._ups = ups
self._queue = []
self._last_rate_adjust = time.time()
__author__ = '[email protected] (Peter Mattis)' import boto import time import urllib from boto.s3.connection import S3Connection from functools import partial from viewfinder.backend.base import constants, counters, util from viewfinder.backend.base.secrets import GetSecret from viewfinder.backend.storage.object_store import ObjectStore from viewfinder.backend.storage.async_s3 import AsyncS3Connection from xml.etree import ElementTree _puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min', 'Average S3 puts per minute.', 60) _secs_per_put = counters.define_average( 'viewfinder.s3.secs_per_put', 'Average time in seconds to complete each S3 put') _gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min', 'Average S3 gets per minute.', 60) class S3ObjectStore(ObjectStore): """Simple object storage interface supporting key/value pairs backed by S3. Methods that require network round-trips are asynchronous; they take a callback argument that will be invoked if the operation completes successfully. If the operation fails, then an exception is raised. To handle this exception, use a Barrier instance with the exception handler defined. """
__author__ = '[email protected] (Spencer Kimball)' import httplib import logging import os import traceback from tornado import gen, web from viewfinder.backend.base import counters, handler from viewfinder.backend.db.db_client import DBClient from viewfinder.backend.db import schema, vf_schema from viewfinder.backend.db.admin_permissions import AdminPermissions from viewfinder.backend.www import basic_auth _req_per_sec = counters.define_rate( 'viewfinder.admin.www.requests_per_second', 'Administrator website requests handled per second.') def require_permission(level=None): """Decorator to be used in admin get/post methods. Permission required may be 'root', 'support', or None. If None is specified, the user must still be in the AdminPermissions table. Permissions are stored in self._permissions for later access. """ def decorator(f): @gen.engine def wrapper(self, *args, **kwargs): assert level in [None, 'root', 'support'] self._permissions = yield gen.Task(self.QueryAdminPermissions)
""" __author__ = '[email protected] (Spencer Kimball)' import httplib import logging import os import traceback from tornado import gen, web from viewfinder.backend.base import counters, handler from viewfinder.backend.db.db_client import DBClient from viewfinder.backend.db import schema, vf_schema from viewfinder.backend.db.admin_permissions import AdminPermissions from viewfinder.backend.www import basic_auth _req_per_sec = counters.define_rate('viewfinder.admin.www.requests_per_second', 'Administrator website requests handled per second.') def require_permission(level=None): """Decorator to be used in admin get/post methods. Permission required may be 'root', 'support', or None. If None is specified, the user must still be in the AdminPermissions table. Permissions are stored in self._permissions for later access. """ def decorator(f): @gen.engine def wrapper(self, *args, **kwargs): assert level in [None, 'root', 'support'] self._permissions = yield gen.Task(self.QueryAdminPermissions)
from tornado.ioloop import IOLoop
from viewfinder.backend.base import counters, message, util
from viewfinder.backend.base.exceptions import FailpointError, InvalidRequestError, LimitExceededError, PermissionError
from viewfinder.backend.base.exceptions import CannotWaitError, NotFoundError, LockFailedError, StopOperationError
from viewfinder.backend.db.lock import Lock
from viewfinder.backend.db.lock_resource_type import LockResourceType
from viewfinder.backend.db.operation import Operation
from viewfinder.backend.op.op_mgr_db_client import OpMgrDBClient
from viewfinder.backend.op.op_context import OpContext
# Performance counters for operation module.
# Average operation time is tracked for all operations - its historical value can be used to measure overall resource usage.
# A rate count of operations attempted and retries attempted - these numbers will indicate if a large number of operations are
# resulting in retry attempts.
_avg_op_time = counters.define_average('viewfinder.operation.avg_op_time', 'Average time in seconds per completed operation.')
_ops_per_min = counters.define_rate('viewfinder.operation.ops_per_min', 'Operations attempted per minute.', 60)
_retries_per_min = counters.define_rate('viewfinder.operation.retries_per_min', 'Operation retries attempted per minute.', 60)
_aborts_per_min = counters.define_rate('viewfinder.operation.aborts_per_min', 'Operations aborted per minute.', 60)
# Tuple of exceptions for which we will abort an operation (not retry).
# Any exception base class included here qualifies all of its subclasses.
_ABORTABLE_EXCEPTIONS = (
PermissionError,
InvalidRequestError,
LimitExceededError,
NotFoundError,
)
# Tuple of exceptions which trigger a retry with a smaller initial backoff.
_SMALLER_RETRY_EXCEPTIONS = (
LockFailedError,