def __init__(self):
self._log = grok_logging.getExtendedLogger(self.__class__.__name__)
self._profiling = grok.app.config.getboolean("debugging", "profiling") or self._log.isEnabledFor(logging.DEBUG)
self._pollInterval = grok.app.config.getfloat("metric_collector", "poll_interval")
self._metricErrorGracePeriod = grok.app.config.getfloat("metric_collector", "metric_error_grace_period")
# Interval for periodic garbage collection of our caches (e.g.,
# self._metricInfoCache and self._resourceInfoCache)
self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2
# Time (unix epoch) when to run the next garbage collection of
# self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a
# quick test of the logic. See self._cacheGarbageCollectionIntervalSec.
self._nextCacheGarbageCollectionTime = 0
# We use this to cache info about metrics that helps us avoid unnecessary
# queries to the datasource. The keys are metric uid's and corresponding
# values are _MetricInfoCacheItem objects.
self._metricInfoCache = defaultdict(_MetricInfoCacheItem)
# We use this to cache info about resources referenced by metrics to help
# us avoid unnecessary resource-status queries to the datasource. The keys
# are resource cananonical identifiers (from Metric.server) and
# corresponding values are _ResourceInfoCacheItem objects.
self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem)
self.metricStreamer = MetricStreamer()
def _collect(task):
""" Executed via multiprocessing Pool: Collect metric data and corresponding
resource status.
:param task: a _DataCollectionTask instance
"""
log = grok_logging.getExtendedLogger(MetricCollector.__name__)
startTime = time.time()
result = _DataCollectionResult(metricID=task.metricID)
dsAdapter = None
try:
dsAdapter = createDatasourceAdapter(task.datasource)
result.data, result.nextCallStart = dsAdapter.getMetricData(
metricSpec=task.metricSpec, start=task.rangeStart, end=None
)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricData failed in task=%s", task)
result.data = e
try:
if task.updateResourceStatus:
result.resourceStatus = dsAdapter.getMetricResourceStatus(metricSpec=task.metricSpec)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricResourceStatus failed in task=%s", task)
result.resourceStatus = e
result.duration = time.time() - startTime
task.resultsQueue.put(result)
return True
def _collect(task):
""" Executed via multiprocessing Pool: Collect metric data and corresponding
resource status.
:param task: a _DataCollectionTask instance
"""
log = grok_logging.getExtendedLogger(MetricCollector.__name__)
startTime = time.time()
result = _DataCollectionResult(metricID=task.metricID)
dsAdapter = None
try:
dsAdapter = createDatasourceAdapter(task.datasource)
result.data, result.nextCallStart = dsAdapter.getMetricData(
metricSpec=task.metricSpec, start=task.rangeStart, end=None)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricData failed in task=%s", task)
result.data = e
try:
if task.updateResourceStatus:
result.resourceStatus = dsAdapter.getMetricResourceStatus(
metricSpec=task.metricSpec)
except Exception as e: # pylint: disable=W0703
log.exception("getMetricResourceStatus failed in task=%s", task)
result.resourceStatus = e
result.duration = time.time() - startTime
task.resultsQueue.put(result)
return True
def __init__(self):
# Make sure we have the latest version of configuration
grok.app.config.loadConfig()
self._log = getExtendedLogger(self.__class__.__name__)
self._modelResultsExchange = (
grok.app.config.get("metric_streamer", "results_exchange_name"))
def __init__(self):
self._log = grok_logging.getExtendedLogger(self.__class__.__name__)
self._profiling = (
grok.app.config.getboolean("debugging", "profiling") or
self._log.isEnabledFor(logging.DEBUG))
self._pollInterval = grok.app.config.getfloat(
"metric_collector", "poll_interval")
self._metricErrorGracePeriod = grok.app.config.getfloat(
"metric_collector", "metric_error_grace_period")
# Interval for periodic garbage collection of our caches (e.g.,
# self._metricInfoCache and self._resourceInfoCache)
self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2
# Time (unix epoch) when to run the next garbage collection of
# self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a
# quick test of the logic. See self._cacheGarbageCollectionIntervalSec.
self._nextCacheGarbageCollectionTime = 0
# We use this to cache info about metrics that helps us avoid unnecessary
# queries to the datasource. The keys are metric uid's and corresponding
# values are _MetricInfoCacheItem objects.
self._metricInfoCache = defaultdict(_MetricInfoCacheItem)
# We use this to cache info about resources referenced by metrics to help
# us avoid unnecessary resource-status queries to the datasource. The keys
# are resource cananonical identifiers (from Metric.server) and
# corresponding values are _ResourceInfoCacheItem objects.
self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem)
self.metricStreamer = MetricStreamer()
def __init__(self):
# Make sure we have the latest version of configuration
grok.app.config.loadConfig()
self._log = getExtendedLogger(self.__class__.__name__)
self._modelResultsExchange = (grok.app.config.get(
"metric_streamer", "results_exchange_name"))
def _getLogger(): return grok_logging.getExtendedLogger(_MODULE_NAME)
def setUpModule():
logging_support.LoggingSupport.initTestApp()
global g_log # pylint: disable=W0603
g_log = getExtendedLogger("autostack_datasource_adapter_test")
import web
import grok
import grok.app
from grok.app import config, repository
from htmengine import utils
from grok.app.repository import schema
from grok.app.aws import s3_utils
from grok import grok_logging
from nta.utils.file_lock import ExclusiveFileLock
# Path format for writing Android logs.
_LOG_FORMAT_ANDROID = os.path.join(os.path.dirname(grok.__file__), "..",
"logs", "android.log")
_LOGGER = grok_logging.getExtendedLogger(__name__)
_AWS_ACCESS_KEY = config.get("aws", "aws_access_key_id")
_AWS_SECRET_KEY = config.get("aws", "aws_secret_access_key")
_BUCKET = "grok.logs"
_MACHINE_ID = uuid.getnode()
_KEY_PREFIX = "metric_dumps/%s-" % _MACHINE_ID
_UPLOAD_ATTEMPTS = 3
urls = (
# /_logging/android
"/android",
"AndroidHandler",
# /_logging/feedback
"/feedback",
"FeedbackHandler",
from grok.app.aws import instance_utils
from grok.app.aws.ec2_utils import checkEC2Authorization
from grok.app.exceptions import AuthFailure, AWSPermissionsError
from grok.app.webservices import (annotations_api, anomalies_api,
autostacks_api, instances_api,
getWebLogPrefix, logging_api, messagemanager,
metrics_api, models_api, notifications_api,
settings_api, support_api, update_api,
wufoo_api, UnauthorizedResponse)
from grok.app.webservices.utils import encodeJson
from htmengine.utils import jsonDecode
logging_support.LoggingSupport.initService()
log = getExtendedLogger("webapp-logger")
urls = (
# Web UI
"",
"DefaultHandler",
"/",
"DefaultHandler",
"/grok",
"GrokHandler",
r"/grok/([-\/\w]*)",
"GrokHandler",
# i18n strings
"/_msgs",
"MessagesHandler",
def _getLogger():
return grok_logging.getExtendedLogger("grok.app.quota")
def _getLogger():
return grok_logging.getExtendedLogger("grok.app.quota")
self._log.exception("RabbitMQ connection failed")
raise
except amqp.exceptions.AmqpChannelError:
self._log.exception("RabbitMQ channel failed")
raise
except Exception as ex:
self._log.exception("Error Streaming data: %s", ex)
raise
except KeyboardInterrupt:
self._log.info(
"Stopping grok Notification Service: KeyboardInterrupt")
finally:
self._log.info("Grok Notification Service is exiting")
if __name__ == "__main__":
g_log = getExtendedLogger("NotificationService")
logging_support.LoggingSupport.initService()
try:
NotificationService().run()
except KeyboardInterrupt:
pass
except Exception as e:
outp = StringIO.StringIO()
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb, file=outp)
outp.seek(0)
g_log.info(outp.read())
g_log.exception("Unexpected Error")
sys.exit(1)
def _collectMetrics(task):
""" Executed via multiprocessing Pool: Collect metric data
:param task: a _MetricCollectionTask object
:returns: a _MetricCollectionTaskResult object
"""
log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
taskResult = _MetricCollectionTaskResult(refID=task.refID,
metricID=task.metricID,
instanceID=task.instanceID)
try:
if task.metricName == "InstanceCount":
timestamp = datetime.utcnow().replace(second=0, microsecond=0)
taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0), )
else:
backoffSec = 0.75
backoffGrowthFactor = 1.5
maxBackoffSec = 5
rawdata = None
while rawdata is None:
try:
cw = cloudwatch.connect_to_region(region_name=task.region,
**getAWSCredentials())
rawdata = cw.get_metric_statistics(
period=task.period,
start_time=task.timeRange.start,
end_time=task.timeRange.end,
metric_name=task.metricName,
namespace="AWS/EC2",
statistics=task.stats,
dimensions=dict(InstanceId=task.instanceID),
unit=task.unit)
except BotoServerError as e:
if e.status == 400 and e.error_code == "Throttling":
# TODO: unit-test
log.info("Throttling: %r", e)
if backoffSec <= maxBackoffSec:
time.sleep(backoffSec)
backoffSec *= backoffGrowthFactor
else:
raise app_exceptions.MetricThrottleError(repr(e))
else:
raise
# Sort the data by timestamp in ascending order
rawdata.sort(key=lambda row: row["Timestamp"])
# Convert raw data to MetricRecord objects
def getValue(rawDataItem, stats):
if isinstance(stats, basestring):
return rawDataItem[stats]
else:
return dict((field, rawDataItem[field]) for field in stats)
taskResult.data = tuple(
MetricRecord(timestamp=rawItem["Timestamp"],
value=getValue(rawItem, task.stats))
for rawItem in rawdata)
except Exception as e: # pylint: disable=W0703
log.exception("Error in task=%r", task)
taskResult.exception = e
finally:
taskResult.duration = time.time() - taskResult.creationTime
return taskResult
import grok
import grok.app
from grok.app import config, repository
from htmengine import utils
from grok.app.repository import schema
from grok.app.aws import s3_utils
from grok import grok_logging
from nta.utils.file_lock import ExclusiveFileLock
# Path format for writing Android logs.
_LOG_FORMAT_ANDROID = os.path.join(os.path.dirname(grok.__file__), "..",
"logs", "android.log")
_LOGGER = grok_logging.getExtendedLogger(__name__)
_AWS_ACCESS_KEY = config.get("aws", "aws_access_key_id")
_AWS_SECRET_KEY = config.get("aws", "aws_secret_access_key")
_BUCKET = "grok.logs"
_MACHINE_ID = uuid.getnode()
_KEY_PREFIX = "metric_dumps/%s-" % _MACHINE_ID
_UPLOAD_ATTEMPTS = 3
urls = (
# /_logging/android
"/android", "AndroidHandler",
# /_logging/feedback
"/feedback", "FeedbackHandler",
)
import htmengine.exceptions as app_exceptions
from grok.app.webservices.utils import getMetricDisplayFields
from grok.app.webservices import (AuthenticatedBaseHandler,
ManagedConnectionWebapp)
from grok.app.webservices.responses import (InvalidRequestResponse,
NotAllowedResponse)
from grok.app.webservices.utils import loadSchema
from grok import grok_logging
from grok.app.quota import (Quota, checkQuotaForCustomMetricAndRaise,
checkQuotaForInstanceAndRaise)
from grok.app.exceptions import QuotaError
_PROCESSING_TIME_PER_RECORD = 0.05 # seconds per record
log = grok_logging.getExtendedLogger("webservices")
urls = (
'',
'ModelHandler',
'/',
'ModelHandler',
'/data',
'MetricDataHandler',
'/data/stats',
'MetricDataStatsHandler',
'/export',
'ModelExportHandler',
'/([-\w]*)',
'ModelHandler',
'/([-\w]*)/data',
def _getInstanceWorkerLogger():
return getExtendedLogger(_MODULE_NAME + ".INSTANCE")
def _getCollectWorkerLogger():
return getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
def _collectMetrics(task):
""" Executed via multiprocessing Pool: Collect metric data
:param task: a _MetricCollectionTask object
:returns: a _MetricCollectionTaskResult object
"""
log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
taskResult = _MetricCollectionTaskResult(refID=task.refID, metricID=task.metricID, instanceID=task.instanceID)
try:
if task.metricName == "InstanceCount":
timestamp = datetime.utcnow().replace(second=0, microsecond=0)
taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0),)
else:
backoffSec = 0.75
backoffGrowthFactor = 1.5
maxBackoffSec = 5
rawdata = None
while rawdata is None:
try:
cw = cloudwatch.connect_to_region(region_name=task.region, **getAWSCredentials())
rawdata = cw.get_metric_statistics(
period=task.period,
start_time=task.timeRange.start,
end_time=task.timeRange.end,
metric_name=task.metricName,
namespace="AWS/EC2",
statistics=task.stats,
dimensions=dict(InstanceId=task.instanceID),
unit=task.unit,
)
except BotoServerError as e:
if e.status == 400 and e.error_code == "Throttling":
# TODO: unit-test
log.info("Throttling: %r", e)
if backoffSec <= maxBackoffSec:
time.sleep(backoffSec)
backoffSec *= backoffGrowthFactor
else:
raise app_exceptions.MetricThrottleError(repr(e))
else:
raise
# Sort the data by timestamp in ascending order
rawdata.sort(key=lambda row: row["Timestamp"])
# Convert raw data to MetricRecord objects
def getValue(rawDataItem, stats):
if isinstance(stats, basestring):
return rawDataItem[stats]
else:
return dict((field, rawDataItem[field]) for field in stats)
taskResult.data = tuple(
MetricRecord(timestamp=rawItem["Timestamp"], value=getValue(rawItem, task.stats)) for rawItem in rawdata
)
except Exception as e: # pylint: disable=W0703
log.exception("Error in task=%r", task)
taskResult.exception = e
finally:
taskResult.duration = time.time() - taskResult.creationTime
return taskResult
self._log.exception("RabbitMQ connection failed")
raise
except amqp.exceptions.AmqpChannelError:
self._log.exception("RabbitMQ channel failed")
raise
except Exception as ex:
self._log.exception("Error Streaming data: %s", ex)
raise
except KeyboardInterrupt:
self._log.info("Stopping grok Notification Service: KeyboardInterrupt")
finally:
self._log.info("Grok Notification Service is exiting")
if __name__ == "__main__":
g_log = getExtendedLogger("NotificationService")
logging_support.LoggingSupport.initService()
try:
NotificationService().run()
except KeyboardInterrupt:
pass
except Exception as e:
outp = StringIO.StringIO()
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb, file=outp)
outp.seek(0)
g_log.info(outp.read())
g_log.exception("Unexpected Error")
sys.exit(1)
def _getCollectWorkerLogger():
return getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
def setUpModule():
logging_support.LoggingSupport.initTestApp()
global g_log # pylint: disable=W0603
g_log = getExtendedLogger("autostack_datasource_adapter_test")
import grok.app
import grok.app.adapters.datasource as datasource_adapter_factory
from grok.app.adapters.datasource.cloudwatch import aws_base
import grok.app.exceptions as app_exceptions
from grok.app import repository
from grok.app.repository import schema
from htmengine.repository.queries import MetricStatus
import htmengine.utils
from grok.test_utils.app.test_case_base import TestCaseBase, unittest
import grok.test_utils.aws_utils
_LOG = getExtendedLogger("cw_datasource_adapter_base_test")
class CloudwatchDatasourceAdapterBaseTest(TestCaseBase):
# EC2 Region name of the test ec2 instance
_testRegion = None
_testId = None
_modelSpecNoMinMax = None
_modelSpecWithMinMax = None
# What we expect to be supported so far
_expectedResourceTypes = [
aws_base.ResourceTypeNames.EC2_INSTANCE,
def _getLogger(): return getExtendedLogger(_MODULE_NAME)
def _getLogger(): return grok_logging.getExtendedLogger(_MODULE_NAME)
metrics_api,
models_api,
notifications_api,
settings_api,
support_api,
update_api,
wufoo_api,
UnauthorizedResponse)
from grok.app.webservices.utils import encodeJson
from htmengine.utils import jsonDecode
logging_support.LoggingSupport.initService()
log = getExtendedLogger("webapp-logger")
urls = (
# Web UI
"", "DefaultHandler",
"/", "DefaultHandler",
"/grok", "GrokHandler",
r"/grok/([-\/\w]*)", "GrokHandler",
# i18n strings
"/_msgs", "MessagesHandler",
# REST API Endpoints
"/_annotations", annotations_api.app,
from grok.grok_logging import getExtendedLogger
import grok.app
import grok.app.adapters.datasource as datasource_adapter_factory
from grok.app.adapters.datasource.cloudwatch import aws_base
from grok.app import repository
from grok.app.repository.queries import MetricStatus
import htmengine.utils
from htmengine.utils import jsonDecode
from grok.test_utils.app.test_case_base import TestCaseBase, unittest
import grok.test_utils.aws_utils
_LOG = getExtendedLogger("cw_datasource_adapter_metrics_test")
def setUpModule():
logging_support.LoggingSupport.initTestApp()
class CloudwatchDatasourceAdapterMetricsTest(TestCaseBase):
_supportedResourceTypes = [
aws_base.ResourceTypeNames.EC2_INSTANCE,
aws_base.ResourceTypeNames.AUTOSCALING_GROUP,
#aws_base.ResourceTypeNames.DYNAMODB_TABLE, (NO DATA)
def _getLogger():
return getExtendedLogger(_MODULE_NAME)
from grok.app.webservices import (AuthenticatedBaseHandler,
ManagedConnectionWebapp)
from grok.app.webservices.models_api import ModelHandler
from grok.app.webservices.responses import (
quotaErrorResponseWrapper,
InvalidRequestResponse)
from grok.app.webservices.utils import encodeJson, loadSchema
# Number of instances to suggest (pre-checked)
_NUM_SUGGESTED_INSTANCES = 8
# Max number of instances to suggest including alternates (unchecked)
_MAX_SUGGESTED_INSTANCES_TOTAL = 28
# Time limit for fetching AWS instances
_AWS_INSTANCE_FETCHING_TIME_LIMIT = 5.0
log = grok_logging.getExtendedLogger("webservices")
urls = (
"/(.+?)/(.+?\/.+?)/(.+\/*.+)", "InstanceDefaultsHandler",
"/(.+)/(.+\/.+)", "InstanceDefaultsHandler",
"/?", "InstancesHandler",
# /_instances/suggestions
"/suggestions", "InstanceSuggestionsHandler",
# /_instances/suggestions/us-west-2
"/suggestions/(.+?)", "InstanceSuggestionsHandler",
)
# Validation schema to ensure we are getting an array of strings.
# Does not validate valid region or namespace, those are handled elsewhere.
_INSTANCES_MODEL_CREATION_SCHEMA = loadSchema(
"instances_model_creation_schema.json")
def _getInstanceWorkerLogger():
return getExtendedLogger(_MODULE_NAME + ".INSTANCE")
from grok import logging_support
from grok.grok_logging import getExtendedLogger
import grok.app
import grok.app.adapters.datasource as datasource_adapter_factory
from grok.app.adapters.datasource.cloudwatch import aws_base
import grok.app.exceptions as app_exceptions
from grok.app import repository
from grok.app.repository import schema
from htmengine.repository.queries import MetricStatus
import htmengine.utils
from grok.test_utils.app.test_case_base import TestCaseBase, unittest
import grok.test_utils.aws_utils
_LOG = getExtendedLogger("cw_datasource_adapter_base_test")
class CloudwatchDatasourceAdapterBaseTest(TestCaseBase):
# EC2 Region name of the test ec2 instance
_testRegion = None
_testId = None
_modelSpecNoMinMax = None
_modelSpecWithMinMax = None
# What we expect to be supported so far
_expectedResourceTypes = [
aws_base.ResourceTypeNames.EC2_INSTANCE,
aws_base.ResourceTypeNames.AUTOSCALING_GROUP,
