コード例 #1
0
ファイル: metric_test.py プロジェクト: zorro0799/viewfinder
  def testMetricsUploadTimer(self):
    """Test the periodic metric upload system."""
    def _OnQueryMetric(min_metrics, max_metrics, metrics):
      self.assertTrue(len(metrics) >= min_metrics and len(metrics) <= max_metrics,
                      '%d not in [%d-%d]' % (len(metrics), min_metrics, max_metrics))
      for m in metrics:
        self.assertTrue(m.timestamp % 3 == 0)
      payload = DotDict(json.loads(metrics[0].payload))
      keys = counters.counters.flatten().keys()
      for k in keys:
        self.assertTrue(k in payload, 'Payload did not contain record for counter %s' % k)
      self.stop()

    start_time = time.time()
    cluster_name = 'test_group_key'
    interval = MetricInterval('test_interval', 3)
    group_key = Metric.EncodeGroupKey(cluster_name, interval)
    rate = counters.define_rate('metricstest.rate', 'Test rate')

    Metric.StartMetricUpload(self._client, cluster_name, interval)
    IOLoop.current().add_timeout(start_time + 7, self.stop)
    rate.increment()
    self.wait(timeout=10)
    end_time = time.time()

    Metric.StopMetricUpload(group_key)
    Metric.QueryTimespan(self._client, group_key, start_time, end_time, partial(_OnQueryMetric, 2, 3))
    Metric.QueryTimespan(self._client, group_key, start_time + 4, end_time, partial(_OnQueryMetric, 1, 2))
    Metric.QueryTimespan(self._client, group_key, start_time, end_time - 4, partial(_OnQueryMetric, 1, 2))
    Metric.QueryTimespan(self._client, group_key, None, end_time, partial(_OnQueryMetric, 2, 3))
    Metric.QueryTimespan(self._client, group_key, start_time, None, partial(_OnQueryMetric, 2, 3))

    # Setting both start_time and end_time to None fails.
    self.assertRaises(AssertionError,
                      Metric.QueryTimespan, self._client, group_key, None, None, partial(_OnQueryMetric, 2, 3))
コード例 #2
0
  def __init__(self, table_name, read_write, name, ups):
    """'ups' is measured either as read or write capacity units per second.
    """
    self._name = name
    self._ups = ups
    self._queue = []
    self._last_rate_adjust = time.time()
    self._unavailable_rate = 0.0
    self._need_adj = False

    qps_counter = backoff_counter = None
    if table_name in kSaveMetricsFor:
      rw_str = 'write' if read_write else 'read'
      qps_counter = counters.define_rate('viewfinder.dynamodb.qps.%s_%s' % (table_name, rw_str),
                                         'Dynamodb %s QPS on %s' % (rw_str, table_name), 1)
      backoff_counter = counters.define_rate('viewfinder.dynamodb.backoff_per_sec.%s_%s' % (table_name, rw_str),
                                             'Dynamodb %s backoff seconds per second on %s' % (rw_str, table_name), 1)
    self._ups_rate = rate_limiter.RateLimiter(ups, qps_counter=qps_counter, backoff_counter=backoff_counter)

    self._timeout = None
コード例 #3
0
__author__ = '[email protected] (Peter Mattis)'

import boto
import time
import urllib

from boto.s3.connection import S3Connection
from functools import partial
from viewfinder.backend.base import constants, counters, util
from viewfinder.backend.base.secrets import GetSecret
from viewfinder.backend.storage.object_store import ObjectStore
from viewfinder.backend.storage.async_s3 import AsyncS3Connection
from xml.etree import ElementTree

_puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min', 'Average S3 puts per minute.', 60)
_secs_per_put = counters.define_average('viewfinder.s3.secs_per_put', 'Average time in seconds to complete each S3 put')
_gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min', 'Average S3 gets per minute.', 60)

class S3ObjectStore(ObjectStore):
  """Simple object storage interface supporting key/value pairs backed by S3.
  Methods that require network round-trips are asynchronous; they take a
  callback argument that will be invoked if the operation completes
  successfully. If the operation fails, then an exception is raised. To
  handle this exception, use a Barrier instance with the exception handler
  defined.
  """
  def __init__(self, bucket_name, temporary=False, read_only=False):
    assert not temporary, 'temporary can only be specified True for file object store'
    self._bucket_name = bucket_name
    self._read_only = read_only
コード例 #4
0
kMaxCapacityThrottleFraction = 0.75

# What fraction of the throughput capacity do we lose every time we receive a throttle from dynamodb.
# eg: 0.05 means that we'll lose 5% of our capacity every time.
kPerThrottleLostCapacityFraction = 0.1

# Minimum amount of time between rate adjustments, in seconds.
kMinRateAdjustmentPeriod = 1.0


DynDBRequest = namedtuple('DynDBRequest', ['method', 'request', 'op', 'execute_cb', 'finish_cb'])

_requests_queued = counters.define_total('viewfinder.dynamodb.requests_queued',
                                         'Number of DynamoDB requests currently queued.')
# TODO: we should have this per table. A global counter is mostly meaningless.
_throttles_per_min = counters.define_rate('viewfinder.dynamodb.throttles_per_min',
                                          'Number of throttling errors received from DynamoDB per minute.', 60)
# In addition to these counters, each RequestQueue may setup an extra two (one for QPS, one for backoff).


class RequestQueue(object):
  """Manages the complexity of tracking successes and failures and
  estimating backoff delays for a request queue.
  """

  def __init__(self, table_name, read_write, name, ups):
    """'ups' is measured either as read or write capacity units per second.
    """
    self._name = name
    self._ups = ups
    self._queue = []
    self._last_rate_adjust = time.time()
コード例 #5
0
__author__ = '[email protected] (Peter Mattis)'

import boto
import time
import urllib

from boto.s3.connection import S3Connection
from functools import partial
from viewfinder.backend.base import constants, counters, util
from viewfinder.backend.base.secrets import GetSecret
from viewfinder.backend.storage.object_store import ObjectStore
from viewfinder.backend.storage.async_s3 import AsyncS3Connection
from xml.etree import ElementTree

_puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min',
                                     'Average S3 puts per minute.', 60)
_secs_per_put = counters.define_average(
    'viewfinder.s3.secs_per_put',
    'Average time in seconds to complete each S3 put')
_gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min',
                                     'Average S3 gets per minute.', 60)


class S3ObjectStore(ObjectStore):
    """Simple object storage interface supporting key/value pairs backed by S3.
  Methods that require network round-trips are asynchronous; they take a
  callback argument that will be invoked if the operation completes
  successfully. If the operation fails, then an exception is raised. To
  handle this exception, use a Barrier instance with the exception handler
  defined.
  """
コード例 #6
0
ファイル: admin.py プロジェクト: zorro0799/viewfinder
__author__ = '[email protected] (Spencer Kimball)'

import httplib
import logging
import os
import traceback
from tornado import gen, web
from viewfinder.backend.base import counters, handler
from viewfinder.backend.db.db_client import DBClient
from viewfinder.backend.db import schema, vf_schema
from viewfinder.backend.db.admin_permissions import AdminPermissions
from viewfinder.backend.www import basic_auth

_req_per_sec = counters.define_rate(
    'viewfinder.admin.www.requests_per_second',
    'Administrator website requests handled per second.')


def require_permission(level=None):
    """Decorator to be used in admin get/post methods.
  Permission required may be 'root', 'support', or None.
  If None is specified, the user must still be in the AdminPermissions table.
  Permissions are stored in self._permissions for later access.
  """
    def decorator(f):
        @gen.engine
        def wrapper(self, *args, **kwargs):
            assert level in [None, 'root', 'support']
            self._permissions = yield gen.Task(self.QueryAdminPermissions)
コード例 #7
0
ファイル: admin.py プロジェクト: 00zhengfu00/viewfinder
"""

__author__ = '[email protected] (Spencer Kimball)'

import httplib
import logging
import os
import traceback
from tornado import gen, web
from viewfinder.backend.base import counters, handler
from viewfinder.backend.db.db_client import DBClient
from viewfinder.backend.db import schema, vf_schema
from viewfinder.backend.db.admin_permissions import AdminPermissions
from viewfinder.backend.www import basic_auth

_req_per_sec = counters.define_rate('viewfinder.admin.www.requests_per_second',
                                    'Administrator website requests handled per second.')



def require_permission(level=None):
  """Decorator to be used in admin get/post methods.
  Permission required may be 'root', 'support', or None.
  If None is specified, the user must still be in the AdminPermissions table.
  Permissions are stored in self._permissions for later access.
  """
  def decorator(f):
    @gen.engine
    def wrapper(self, *args, **kwargs):
      assert level in [None, 'root', 'support']
      self._permissions = yield gen.Task(self.QueryAdminPermissions)
コード例 #8
0
from tornado.ioloop import IOLoop
from viewfinder.backend.base import counters, message, util
from viewfinder.backend.base.exceptions import FailpointError, InvalidRequestError, LimitExceededError, PermissionError
from viewfinder.backend.base.exceptions import CannotWaitError, NotFoundError, LockFailedError, StopOperationError
from viewfinder.backend.db.lock import Lock
from viewfinder.backend.db.lock_resource_type import LockResourceType
from viewfinder.backend.db.operation import Operation
from viewfinder.backend.op.op_mgr_db_client import OpMgrDBClient
from viewfinder.backend.op.op_context import OpContext

# Performance counters for operation module.
# Average operation time is tracked for all operations - its historical value can be used to measure overall resource usage.
# A rate count of operations attempted and retries attempted - these numbers will indicate if a large number of operations are
# resulting in retry attempts.
_avg_op_time = counters.define_average('viewfinder.operation.avg_op_time', 'Average time in seconds per completed operation.')
_ops_per_min = counters.define_rate('viewfinder.operation.ops_per_min', 'Operations attempted per minute.', 60)
_retries_per_min = counters.define_rate('viewfinder.operation.retries_per_min', 'Operation retries attempted per minute.', 60)
_aborts_per_min = counters.define_rate('viewfinder.operation.aborts_per_min', 'Operations aborted per minute.', 60)

# Tuple of exceptions for which we will abort an operation (not retry).
# Any exception base class included here qualifies all of its subclasses.
_ABORTABLE_EXCEPTIONS = (
  PermissionError,
  InvalidRequestError,
  LimitExceededError,
  NotFoundError,
  )

# Tuple of exceptions which trigger a retry with a smaller initial backoff.
_SMALLER_RETRY_EXCEPTIONS = (
  LockFailedError,