Esempio n. 1
0
  def __init__(self):
    self._log = YOMP_logging.getExtendedLogger(self.__class__.__name__)

    self._profiling = (
      YOMP.app.config.getboolean("debugging", "profiling") or
      self._log.isEnabledFor(logging.DEBUG))

    self._pollInterval = YOMP.app.config.getfloat(
      "metric_collector", "poll_interval")

    self._metricErrorGracePeriod = YOMP.app.config.getfloat(
      "metric_collector", "metric_error_grace_period")

    # Interval for periodic garbage collection of our caches (e.g.,
    # self._metricInfoCache and self._resourceInfoCache)
    self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2

    # Time (unix epoch) when to run the next garbage collection of
    # self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a
    # quick test of the logic. See self._cacheGarbageCollectionIntervalSec.
    self._nextCacheGarbageCollectionTime = 0

    # We use this to cache info about metrics that helps us avoid unnecessary
    # queries to the datasource. The keys are metric uid's and corresponding
    # values are _MetricInfoCacheItem objects.
    self._metricInfoCache = defaultdict(_MetricInfoCacheItem)

    # We use this to cache info about resources referenced by metrics to help
    # us avoid unnecessary resource-status queries to the datasource. The keys
    # are resource cananonical identifiers (from Metric.server) and
    # corresponding values are _ResourceInfoCacheItem objects.
    self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem)

    self.metricStreamer = MetricStreamer()
Esempio n. 2
0
def _collect(task):
    """ Executed via multiprocessing Pool: Collect metric data and corresponding
  resource status.

  :param task: a _DataCollectionTask instance
  """
    log = YOMP_logging.getExtendedLogger(MetricCollector.__name__)

    startTime = time.time()

    result = _DataCollectionResult(metricID=task.metricID)

    dsAdapter = None

    try:
        dsAdapter = createDatasourceAdapter(task.datasource)
        result.data, result.nextCallStart = dsAdapter.getMetricData(
            metricSpec=task.metricSpec, start=task.rangeStart, end=None)
    except Exception as e:  # pylint: disable=W0703
        log.exception("getMetricData failed in task=%s", task)
        result.data = e

    try:
        if task.updateResourceStatus:
            result.resourceStatus = dsAdapter.getMetricResourceStatus(
                metricSpec=task.metricSpec)
    except Exception as e:  # pylint: disable=W0703
        log.exception("getMetricResourceStatus failed in task=%s", task)
        result.resourceStatus = e

    result.duration = time.time() - startTime

    task.resultsQueue.put(result)

    return True
Esempio n. 3
0
    def __init__(self):
        self._log = YOMP_logging.getExtendedLogger(self.__class__.__name__)

        self._profiling = (YOMP.app.config.getboolean("debugging", "profiling")
                           or self._log.isEnabledFor(logging.DEBUG))

        self._pollInterval = YOMP.app.config.getfloat("metric_collector",
                                                      "poll_interval")

        self._metricErrorGracePeriod = YOMP.app.config.getfloat(
            "metric_collector", "metric_error_grace_period")

        # Interval for periodic garbage collection of our caches (e.g.,
        # self._metricInfoCache and self._resourceInfoCache)
        self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2

        # Time (unix epoch) when to run the next garbage collection of
        # self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a
        # quick test of the logic. See self._cacheGarbageCollectionIntervalSec.
        self._nextCacheGarbageCollectionTime = 0

        # We use this to cache info about metrics that helps us avoid unnecessary
        # queries to the datasource. The keys are metric uid's and corresponding
        # values are _MetricInfoCacheItem objects.
        self._metricInfoCache = defaultdict(_MetricInfoCacheItem)

        # We use this to cache info about resources referenced by metrics to help
        # us avoid unnecessary resource-status queries to the datasource. The keys
        # are resource cananonical identifiers (from Metric.server) and
        # corresponding values are _ResourceInfoCacheItem objects.
        self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem)

        self.metricStreamer = MetricStreamer()
Esempio n. 4
0
def _collect(task):
  """ Executed via multiprocessing Pool: Collect metric data and corresponding
  resource status.

  :param task: a _DataCollectionTask instance
  """
  log = YOMP_logging.getExtendedLogger(MetricCollector.__name__)

  startTime = time.time()

  result = _DataCollectionResult(metricID=task.metricID)

  dsAdapter = None

  try:
    dsAdapter = createDatasourceAdapter(task.datasource)
    result.data, result.nextCallStart = dsAdapter.getMetricData(
      metricSpec=task.metricSpec,
      start=task.rangeStart,
      end=None)
  except Exception as e: # pylint: disable=W0703
    log.exception("getMetricData failed in task=%s", task)
    result.data = e

  try:
    if task.updateResourceStatus:
      result.resourceStatus = dsAdapter.getMetricResourceStatus(
        metricSpec=task.metricSpec)
  except Exception as e: # pylint: disable=W0703
    log.exception("getMetricResourceStatus failed in task=%s", task)
    result.resourceStatus = e

  result.duration = time.time() - startTime

  task.resultsQueue.put(result)

  return True
Esempio n. 5
0
def _getLogger():
  return YOMP_logging.getExtendedLogger("YOMP.app.quota")
Esempio n. 6
0
from YOMP.app.webservices import (AuthenticatedBaseHandler,
                                  ManagedConnectionWebapp)
from YOMP.app.webservices.models_api import ModelHandler
from YOMP.app.webservices.responses import (
    quotaErrorResponseWrapper,
    InvalidRequestResponse)
from YOMP.app.webservices.utils import encodeJson, loadSchema

# Number of instances to suggest (pre-checked)
_NUM_SUGGESTED_INSTANCES = 8
# Max number of instances to suggest including alternates (unchecked)
_MAX_SUGGESTED_INSTANCES_TOTAL = 28
# Time limit for fetching AWS instances
_AWS_INSTANCE_FETCHING_TIME_LIMIT = 5.0

log = YOMP_logging.getExtendedLogger("webservices")

urls = (
  "/(.+?)/(.+?\/.+?)/(.+\/*.+)", "InstanceDefaultsHandler",
  "/(.+)/(.+\/.+)", "InstanceDefaultsHandler",
  "/?", "InstancesHandler",
  # /_instances/suggestions
  "/suggestions", "InstanceSuggestionsHandler",
  # /_instances/suggestions/us-west-2
  "/suggestions/(.+?)", "InstanceSuggestionsHandler",
)

# Validation schema to ensure we are getting an array of strings.
# Does not validate valid region or namespace, those are handled elsewhere.
_INSTANCES_MODEL_CREATION_SCHEMA = loadSchema(
  "instances_model_creation_schema.json")
Esempio n. 7
0
import web

import YOMP
import YOMP.app
from YOMP.app import config, repository
from htmengine import utils
from YOMP.app.repository import schema
from YOMP.app.aws import s3_utils
from YOMP import YOMP_logging

from nta.utils.file_lock import ExclusiveFileLock


# Path format for writing Android logs.
_LOG_FORMAT_ANDROID = os.path.join(os.path.dirname(YOMP.__file__), "..", "logs", "android.log")
_LOGGER = YOMP_logging.getExtendedLogger(__name__)

_AWS_ACCESS_KEY = config.get("aws", "aws_access_key_id")
_AWS_SECRET_KEY = config.get("aws", "aws_secret_access_key")
_BUCKET = "YOMP.logs"
_MACHINE_ID = uuid.getnode()
_KEY_PREFIX = "metric_dumps/%s-" % _MACHINE_ID
_UPLOAD_ATTEMPTS = 3

urls = (
    # /_logging/android
    "/android",
    "AndroidHandler",
    # /_logging/feedback
    "/feedback",
    "FeedbackHandler",
Esempio n. 8
0
def _getLogger():
  return YOMP_logging.getExtendedLogger(_MODULE_NAME)
Esempio n. 9
0
from YOMP.app.aws import asg_utils, ec2_utils, elb_utils, rds_utils
from YOMP.app.webservices import (AuthenticatedBaseHandler,
                                  ManagedConnectionWebapp)
from YOMP.app.webservices.models_api import ModelHandler
from YOMP.app.webservices.responses import (quotaErrorResponseWrapper,
                                            InvalidRequestResponse)
from YOMP.app.webservices.utils import encodeJson, loadSchema

# Number of instances to suggest (pre-checked)
_NUM_SUGGESTED_INSTANCES = 8
# Max number of instances to suggest including alternates (unchecked)
_MAX_SUGGESTED_INSTANCES_TOTAL = 28
# Time limit for fetching AWS instances
_AWS_INSTANCE_FETCHING_TIME_LIMIT = 5.0

log = YOMP_logging.getExtendedLogger("webservices")

urls = (
    "/(.+?)/(.+?\/.+?)/(.+\/*.+)",
    "InstanceDefaultsHandler",
    "/(.+)/(.+\/.+)",
    "InstanceDefaultsHandler",
    "/?",
    "InstancesHandler",
    # /_instances/suggestions
    "/suggestions",
    "InstanceSuggestionsHandler",
    # /_instances/suggestions/us-west-2
    "/suggestions/(.+?)",
    "InstanceSuggestionsHandler",
)
Esempio n. 10
0
def _getLogger():
    return YOMP_logging.getExtendedLogger(_MODULE_NAME)
Esempio n. 11
0
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import json

import web

from YOMP.app.adapters.datasource import createDatasourceAdapter
from YOMP.app import exceptions as app_exceptions, repository
from YOMP.app.webservices import (AuthenticatedBaseHandler,
                                  ManagedConnectionWebapp)
from YOMP.app.webservices.utils import (getMetricDisplayFields,
                                        convertMetricRowToMetricDict)
from YOMP import YOMP_logging

LOGGER = YOMP_logging.getExtendedLogger(__name__)

urls = (
  # /_metrics/custom
  '', "CustomDefaultHandler",
  # /_metrics/custom/
  '/', "CustomDefaultHandler",
  # /_metrics/custom/<metricName>
  '/([\w\.\-]+)', "CustomDefaultHandler",
)



class CustomDefaultHandler(AuthenticatedBaseHandler):

Esempio n. 12
0
def _getLogger():
    return YOMP_logging.getExtendedLogger("YOMP.app.quota")