def __init__(self):
self._log = YOMP_logging.getExtendedLogger(self.__class__.__name__)
self._profiling = (
YOMP.app.config.getboolean("debugging", "profiling") or
self._log.isEnabledFor(logging.DEBUG))
self._pollInterval = YOMP.app.config.getfloat(
"metric_collector", "poll_interval")
self._metricErrorGracePeriod = YOMP.app.config.getfloat(
"metric_collector", "metric_error_grace_period")
# Interval for periodic garbage collection of our caches (e.g.,
# self._metricInfoCache and self._resourceInfoCache)
self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2
# Time (unix epoch) when to run the next garbage collection of
# self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a
# quick test of the logic. See self._cacheGarbageCollectionIntervalSec.
self._nextCacheGarbageCollectionTime = 0
# We use this to cache info about metrics that helps us avoid unnecessary
# queries to the datasource. The keys are metric uid's and corresponding
# values are _MetricInfoCacheItem objects.
self._metricInfoCache = defaultdict(_MetricInfoCacheItem)
# We use this to cache info about resources referenced by metrics to help
# us avoid unnecessary resource-status queries to the datasource. The keys
# are resource cananonical identifiers (from Metric.server) and
# corresponding values are _ResourceInfoCacheItem objects.
self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem)
self.metricStreamer = MetricStreamer()
def _collect(task):
""" Executed via multiprocessing Pool: Collect metric data and corresponding
resource status.
:param task: a _DataCollectionTask instance
"""
log = YOMP_logging.getExtendedLogger(MetricCollector.__name__)
startTime = time.time()
result = _DataCollectionResult(metricID=task.metricID)
dsAdapter = None
try:
dsAdapter = createDatasourceAdapter(task.datasource)
result.data, result.nextCallStart = dsAdapter.getMetricData(
metricSpec=task.metricSpec, start=task.rangeStart, end=None)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricData failed in task=%s", task)
result.data = e
try:
if task.updateResourceStatus:
result.resourceStatus = dsAdapter.getMetricResourceStatus(
metricSpec=task.metricSpec)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricResourceStatus failed in task=%s", task)
result.resourceStatus = e
result.duration = time.time() - startTime
task.resultsQueue.put(result)
return True
def __init__(self):
self._log = YOMP_logging.getExtendedLogger(self.__class__.__name__)
self._profiling = (YOMP.app.config.getboolean("debugging", "profiling")
or self._log.isEnabledFor(logging.DEBUG))
self._pollInterval = YOMP.app.config.getfloat("metric_collector",
"poll_interval")
self._metricErrorGracePeriod = YOMP.app.config.getfloat(
"metric_collector", "metric_error_grace_period")
# Interval for periodic garbage collection of our caches (e.g.,
# self._metricInfoCache and self._resourceInfoCache)
self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2
# Time (unix epoch) when to run the next garbage collection of
# self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a
# quick test of the logic. See self._cacheGarbageCollectionIntervalSec.
self._nextCacheGarbageCollectionTime = 0
# We use this to cache info about metrics that helps us avoid unnecessary
# queries to the datasource. The keys are metric uid's and corresponding
# values are _MetricInfoCacheItem objects.
self._metricInfoCache = defaultdict(_MetricInfoCacheItem)
# We use this to cache info about resources referenced by metrics to help
# us avoid unnecessary resource-status queries to the datasource. The keys
# are resource cananonical identifiers (from Metric.server) and
# corresponding values are _ResourceInfoCacheItem objects.
self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem)
self.metricStreamer = MetricStreamer()
def _collect(task):
""" Executed via multiprocessing Pool: Collect metric data and corresponding
resource status.
:param task: a _DataCollectionTask instance
"""
log = YOMP_logging.getExtendedLogger(MetricCollector.__name__)
startTime = time.time()
result = _DataCollectionResult(metricID=task.metricID)
dsAdapter = None
try:
dsAdapter = createDatasourceAdapter(task.datasource)
result.data, result.nextCallStart = dsAdapter.getMetricData(
metricSpec=task.metricSpec,
start=task.rangeStart,
end=None)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricData failed in task=%s", task)
result.data = e
try:
if task.updateResourceStatus:
result.resourceStatus = dsAdapter.getMetricResourceStatus(
metricSpec=task.metricSpec)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricResourceStatus failed in task=%s", task)
result.resourceStatus = e
result.duration = time.time() - startTime
task.resultsQueue.put(result)
return True
def _getLogger():
return YOMP_logging.getExtendedLogger("YOMP.app.quota")
from YOMP.app.webservices import (AuthenticatedBaseHandler,
ManagedConnectionWebapp)
from YOMP.app.webservices.models_api import ModelHandler
from YOMP.app.webservices.responses import (
quotaErrorResponseWrapper,
InvalidRequestResponse)
from YOMP.app.webservices.utils import encodeJson, loadSchema
# Number of instances to suggest (pre-checked)
_NUM_SUGGESTED_INSTANCES = 8
# Max number of instances to suggest including alternates (unchecked)
_MAX_SUGGESTED_INSTANCES_TOTAL = 28
# Time limit for fetching AWS instances
_AWS_INSTANCE_FETCHING_TIME_LIMIT = 5.0
log = YOMP_logging.getExtendedLogger("webservices")
urls = (
"/(.+?)/(.+?\/.+?)/(.+\/*.+)", "InstanceDefaultsHandler",
"/(.+)/(.+\/.+)", "InstanceDefaultsHandler",
"/?", "InstancesHandler",
# /_instances/suggestions
"/suggestions", "InstanceSuggestionsHandler",
# /_instances/suggestions/us-west-2
"/suggestions/(.+?)", "InstanceSuggestionsHandler",
)
# Validation schema to ensure we are getting an array of strings.
# Does not validate valid region or namespace, those are handled elsewhere.
_INSTANCES_MODEL_CREATION_SCHEMA = loadSchema(
"instances_model_creation_schema.json")
import web
import YOMP
import YOMP.app
from YOMP.app import config, repository
from htmengine import utils
from YOMP.app.repository import schema
from YOMP.app.aws import s3_utils
from YOMP import YOMP_logging
from nta.utils.file_lock import ExclusiveFileLock
# Path format for writing Android logs.
_LOG_FORMAT_ANDROID = os.path.join(os.path.dirname(YOMP.__file__), "..", "logs", "android.log")
_LOGGER = YOMP_logging.getExtendedLogger(__name__)
_AWS_ACCESS_KEY = config.get("aws", "aws_access_key_id")
_AWS_SECRET_KEY = config.get("aws", "aws_secret_access_key")
_BUCKET = "YOMP.logs"
_MACHINE_ID = uuid.getnode()
_KEY_PREFIX = "metric_dumps/%s-" % _MACHINE_ID
_UPLOAD_ATTEMPTS = 3
urls = (
# /_logging/android
"/android",
"AndroidHandler",
# /_logging/feedback
"/feedback",
"FeedbackHandler",
def _getLogger(): return YOMP_logging.getExtendedLogger(_MODULE_NAME)
from YOMP.app.aws import asg_utils, ec2_utils, elb_utils, rds_utils
from YOMP.app.webservices import (AuthenticatedBaseHandler,
ManagedConnectionWebapp)
from YOMP.app.webservices.models_api import ModelHandler
from YOMP.app.webservices.responses import (quotaErrorResponseWrapper,
InvalidRequestResponse)
from YOMP.app.webservices.utils import encodeJson, loadSchema
# Number of instances to suggest (pre-checked)
_NUM_SUGGESTED_INSTANCES = 8
# Max number of instances to suggest including alternates (unchecked)
_MAX_SUGGESTED_INSTANCES_TOTAL = 28
# Time limit for fetching AWS instances
_AWS_INSTANCE_FETCHING_TIME_LIMIT = 5.0
log = YOMP_logging.getExtendedLogger("webservices")
urls = (
"/(.+?)/(.+?\/.+?)/(.+\/*.+)",
"InstanceDefaultsHandler",
"/(.+)/(.+\/.+)",
"InstanceDefaultsHandler",
"/?",
"InstancesHandler",
# /_instances/suggestions
"/suggestions",
"InstanceSuggestionsHandler",
# /_instances/suggestions/us-west-2
"/suggestions/(.+?)",
"InstanceSuggestionsHandler",
)
def _getLogger():
return YOMP_logging.getExtendedLogger(_MODULE_NAME)
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
import json
import web
from YOMP.app.adapters.datasource import createDatasourceAdapter
from YOMP.app import exceptions as app_exceptions, repository
from YOMP.app.webservices import (AuthenticatedBaseHandler,
ManagedConnectionWebapp)
from YOMP.app.webservices.utils import (getMetricDisplayFields,
convertMetricRowToMetricDict)
from YOMP import YOMP_logging
LOGGER = YOMP_logging.getExtendedLogger(__name__)
urls = (
# /_metrics/custom
'', "CustomDefaultHandler",
# /_metrics/custom/
'/', "CustomDefaultHandler",
# /_metrics/custom/<metricName>
'/([\w\.\-]+)', "CustomDefaultHandler",
)
class CustomDefaultHandler(AuthenticatedBaseHandler):
def _getLogger():
return YOMP_logging.getExtendedLogger("YOMP.app.quota")