def __init__(self): self._log = grok_logging.getExtendedLogger(self.__class__.__name__) self._profiling = grok.app.config.getboolean("debugging", "profiling") or self._log.isEnabledFor(logging.DEBUG) self._pollInterval = grok.app.config.getfloat("metric_collector", "poll_interval") self._metricErrorGracePeriod = grok.app.config.getfloat("metric_collector", "metric_error_grace_period") # Interval for periodic garbage collection of our caches (e.g., # self._metricInfoCache and self._resourceInfoCache) self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2 # Time (unix epoch) when to run the next garbage collection of # self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a # quick test of the logic. See self._cacheGarbageCollectionIntervalSec. self._nextCacheGarbageCollectionTime = 0 # We use this to cache info about metrics that helps us avoid unnecessary # queries to the datasource. The keys are metric uid's and corresponding # values are _MetricInfoCacheItem objects. self._metricInfoCache = defaultdict(_MetricInfoCacheItem) # We use this to cache info about resources referenced by metrics to help # us avoid unnecessary resource-status queries to the datasource. The keys # are resource cananonical identifiers (from Metric.server) and # corresponding values are _ResourceInfoCacheItem objects. self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem) self.metricStreamer = MetricStreamer()
def _collect(task): """ Executed via multiprocessing Pool: Collect metric data and corresponding resource status. :param task: a _DataCollectionTask instance """ log = grok_logging.getExtendedLogger(MetricCollector.__name__) startTime = time.time() result = _DataCollectionResult(metricID=task.metricID) dsAdapter = None try: dsAdapter = createDatasourceAdapter(task.datasource) result.data, result.nextCallStart = dsAdapter.getMetricData( metricSpec=task.metricSpec, start=task.rangeStart, end=None ) except Exception as e: # pylint: disable=W0703 log.exception("getMetricData failed in task=%s", task) result.data = e try: if task.updateResourceStatus: result.resourceStatus = dsAdapter.getMetricResourceStatus(metricSpec=task.metricSpec) except Exception as e: # pylint: disable=W0703 log.exception("getMetricResourceStatus failed in task=%s", task) result.resourceStatus = e result.duration = time.time() - startTime task.resultsQueue.put(result) return True
def _collect(task): """ Executed via multiprocessing Pool: Collect metric data and corresponding resource status. :param task: a _DataCollectionTask instance """ log = grok_logging.getExtendedLogger(MetricCollector.__name__) startTime = time.time() result = _DataCollectionResult(metricID=task.metricID) dsAdapter = None try: dsAdapter = createDatasourceAdapter(task.datasource) result.data, result.nextCallStart = dsAdapter.getMetricData( metricSpec=task.metricSpec, start=task.rangeStart, end=None) except Exception as e: # pylint: disable=W0703 log.exception("getMetricData failed in task=%s", task) result.data = e try: if task.updateResourceStatus: result.resourceStatus = dsAdapter.getMetricResourceStatus( metricSpec=task.metricSpec) except Exception as e: # pylint: disable=W0703 log.exception("getMetricResourceStatus failed in task=%s", task) result.resourceStatus = e result.duration = time.time() - startTime task.resultsQueue.put(result) return True
def __init__(self): # Make sure we have the latest version of configuration grok.app.config.loadConfig() self._log = getExtendedLogger(self.__class__.__name__) self._modelResultsExchange = ( grok.app.config.get("metric_streamer", "results_exchange_name"))
def __init__(self): self._log = grok_logging.getExtendedLogger(self.__class__.__name__) self._profiling = ( grok.app.config.getboolean("debugging", "profiling") or self._log.isEnabledFor(logging.DEBUG)) self._pollInterval = grok.app.config.getfloat( "metric_collector", "poll_interval") self._metricErrorGracePeriod = grok.app.config.getfloat( "metric_collector", "metric_error_grace_period") # Interval for periodic garbage collection of our caches (e.g., # self._metricInfoCache and self._resourceInfoCache) self._cacheGarbageCollectionIntervalSec = self._metricErrorGracePeriod * 2 # Time (unix epoch) when to run the next garbage collection of # self._metricInfoCache and self._resourceInfoCache; 0 triggers it ASAP as a # quick test of the logic. See self._cacheGarbageCollectionIntervalSec. self._nextCacheGarbageCollectionTime = 0 # We use this to cache info about metrics that helps us avoid unnecessary # queries to the datasource. The keys are metric uid's and corresponding # values are _MetricInfoCacheItem objects. self._metricInfoCache = defaultdict(_MetricInfoCacheItem) # We use this to cache info about resources referenced by metrics to help # us avoid unnecessary resource-status queries to the datasource. The keys # are resource cananonical identifiers (from Metric.server) and # corresponding values are _ResourceInfoCacheItem objects. self._resourceInfoCache = defaultdict(_ResourceInfoCacheItem) self.metricStreamer = MetricStreamer()
def __init__(self): # Make sure we have the latest version of configuration grok.app.config.loadConfig() self._log = getExtendedLogger(self.__class__.__name__) self._modelResultsExchange = (grok.app.config.get( "metric_streamer", "results_exchange_name"))
def _getLogger(): return grok_logging.getExtendedLogger(_MODULE_NAME)
def setUpModule(): logging_support.LoggingSupport.initTestApp() global g_log # pylint: disable=W0603 g_log = getExtendedLogger("autostack_datasource_adapter_test")
import web import grok import grok.app from grok.app import config, repository from htmengine import utils from grok.app.repository import schema from grok.app.aws import s3_utils from grok import grok_logging from nta.utils.file_lock import ExclusiveFileLock # Path format for writing Android logs. _LOG_FORMAT_ANDROID = os.path.join(os.path.dirname(grok.__file__), "..", "logs", "android.log") _LOGGER = grok_logging.getExtendedLogger(__name__) _AWS_ACCESS_KEY = config.get("aws", "aws_access_key_id") _AWS_SECRET_KEY = config.get("aws", "aws_secret_access_key") _BUCKET = "grok.logs" _MACHINE_ID = uuid.getnode() _KEY_PREFIX = "metric_dumps/%s-" % _MACHINE_ID _UPLOAD_ATTEMPTS = 3 urls = ( # /_logging/android "/android", "AndroidHandler", # /_logging/feedback "/feedback", "FeedbackHandler",
from grok.app.aws import instance_utils from grok.app.aws.ec2_utils import checkEC2Authorization from grok.app.exceptions import AuthFailure, AWSPermissionsError from grok.app.webservices import (annotations_api, anomalies_api, autostacks_api, instances_api, getWebLogPrefix, logging_api, messagemanager, metrics_api, models_api, notifications_api, settings_api, support_api, update_api, wufoo_api, UnauthorizedResponse) from grok.app.webservices.utils import encodeJson from htmengine.utils import jsonDecode logging_support.LoggingSupport.initService() log = getExtendedLogger("webapp-logger") urls = ( # Web UI "", "DefaultHandler", "/", "DefaultHandler", "/grok", "GrokHandler", r"/grok/([-\/\w]*)", "GrokHandler", # i18n strings "/_msgs", "MessagesHandler",
def _getLogger(): return grok_logging.getExtendedLogger("grok.app.quota")
self._log.exception("RabbitMQ connection failed") raise except amqp.exceptions.AmqpChannelError: self._log.exception("RabbitMQ channel failed") raise except Exception as ex: self._log.exception("Error Streaming data: %s", ex) raise except KeyboardInterrupt: self._log.info( "Stopping grok Notification Service: KeyboardInterrupt") finally: self._log.info("Grok Notification Service is exiting") if __name__ == "__main__": g_log = getExtendedLogger("NotificationService") logging_support.LoggingSupport.initService() try: NotificationService().run() except KeyboardInterrupt: pass except Exception as e: outp = StringIO.StringIO() ex_type, ex, tb = sys.exc_info() traceback.print_tb(tb, file=outp) outp.seek(0) g_log.info(outp.read()) g_log.exception("Unexpected Error") sys.exit(1)
def _collectMetrics(task): """ Executed via multiprocessing Pool: Collect metric data :param task: a _MetricCollectionTask object :returns: a _MetricCollectionTaskResult object """ log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR") taskResult = _MetricCollectionTaskResult(refID=task.refID, metricID=task.metricID, instanceID=task.instanceID) try: if task.metricName == "InstanceCount": timestamp = datetime.utcnow().replace(second=0, microsecond=0) taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0), ) else: backoffSec = 0.75 backoffGrowthFactor = 1.5 maxBackoffSec = 5 rawdata = None while rawdata is None: try: cw = cloudwatch.connect_to_region(region_name=task.region, **getAWSCredentials()) rawdata = cw.get_metric_statistics( period=task.period, start_time=task.timeRange.start, end_time=task.timeRange.end, metric_name=task.metricName, namespace="AWS/EC2", statistics=task.stats, dimensions=dict(InstanceId=task.instanceID), unit=task.unit) except BotoServerError as e: if e.status == 400 and e.error_code == "Throttling": # TODO: unit-test log.info("Throttling: %r", e) if backoffSec <= maxBackoffSec: time.sleep(backoffSec) backoffSec *= backoffGrowthFactor else: raise app_exceptions.MetricThrottleError(repr(e)) else: raise # Sort the data by timestamp in ascending order rawdata.sort(key=lambda row: row["Timestamp"]) # Convert raw data to MetricRecord objects def getValue(rawDataItem, stats): if isinstance(stats, basestring): return rawDataItem[stats] else: return dict((field, rawDataItem[field]) for field in stats) taskResult.data = tuple( MetricRecord(timestamp=rawItem["Timestamp"], value=getValue(rawItem, task.stats)) for rawItem in rawdata) except Exception as e: # pylint: disable=W0703 log.exception("Error in task=%r", task) taskResult.exception = e finally: taskResult.duration = time.time() - taskResult.creationTime return taskResult
import grok import grok.app from grok.app import config, repository from htmengine import utils from grok.app.repository import schema from grok.app.aws import s3_utils from grok import grok_logging from nta.utils.file_lock import ExclusiveFileLock # Path format for writing Android logs. _LOG_FORMAT_ANDROID = os.path.join(os.path.dirname(grok.__file__), "..", "logs", "android.log") _LOGGER = grok_logging.getExtendedLogger(__name__) _AWS_ACCESS_KEY = config.get("aws", "aws_access_key_id") _AWS_SECRET_KEY = config.get("aws", "aws_secret_access_key") _BUCKET = "grok.logs" _MACHINE_ID = uuid.getnode() _KEY_PREFIX = "metric_dumps/%s-" % _MACHINE_ID _UPLOAD_ATTEMPTS = 3 urls = ( # /_logging/android "/android", "AndroidHandler", # /_logging/feedback "/feedback", "FeedbackHandler", )
import htmengine.exceptions as app_exceptions from grok.app.webservices.utils import getMetricDisplayFields from grok.app.webservices import (AuthenticatedBaseHandler, ManagedConnectionWebapp) from grok.app.webservices.responses import (InvalidRequestResponse, NotAllowedResponse) from grok.app.webservices.utils import loadSchema from grok import grok_logging from grok.app.quota import (Quota, checkQuotaForCustomMetricAndRaise, checkQuotaForInstanceAndRaise) from grok.app.exceptions import QuotaError _PROCESSING_TIME_PER_RECORD = 0.05 # seconds per record log = grok_logging.getExtendedLogger("webservices") urls = ( '', 'ModelHandler', '/', 'ModelHandler', '/data', 'MetricDataHandler', '/data/stats', 'MetricDataStatsHandler', '/export', 'ModelExportHandler', '/([-\w]*)', 'ModelHandler', '/([-\w]*)/data',
def _getInstanceWorkerLogger(): return getExtendedLogger(_MODULE_NAME + ".INSTANCE")
def _getCollectWorkerLogger(): return getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
def _collectMetrics(task): """ Executed via multiprocessing Pool: Collect metric data :param task: a _MetricCollectionTask object :returns: a _MetricCollectionTaskResult object """ log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR") taskResult = _MetricCollectionTaskResult(refID=task.refID, metricID=task.metricID, instanceID=task.instanceID) try: if task.metricName == "InstanceCount": timestamp = datetime.utcnow().replace(second=0, microsecond=0) taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0),) else: backoffSec = 0.75 backoffGrowthFactor = 1.5 maxBackoffSec = 5 rawdata = None while rawdata is None: try: cw = cloudwatch.connect_to_region(region_name=task.region, **getAWSCredentials()) rawdata = cw.get_metric_statistics( period=task.period, start_time=task.timeRange.start, end_time=task.timeRange.end, metric_name=task.metricName, namespace="AWS/EC2", statistics=task.stats, dimensions=dict(InstanceId=task.instanceID), unit=task.unit, ) except BotoServerError as e: if e.status == 400 and e.error_code == "Throttling": # TODO: unit-test log.info("Throttling: %r", e) if backoffSec <= maxBackoffSec: time.sleep(backoffSec) backoffSec *= backoffGrowthFactor else: raise app_exceptions.MetricThrottleError(repr(e)) else: raise # Sort the data by timestamp in ascending order rawdata.sort(key=lambda row: row["Timestamp"]) # Convert raw data to MetricRecord objects def getValue(rawDataItem, stats): if isinstance(stats, basestring): return rawDataItem[stats] else: return dict((field, rawDataItem[field]) for field in stats) taskResult.data = tuple( MetricRecord(timestamp=rawItem["Timestamp"], value=getValue(rawItem, task.stats)) for rawItem in rawdata ) except Exception as e: # pylint: disable=W0703 log.exception("Error in task=%r", task) taskResult.exception = e finally: taskResult.duration = time.time() - taskResult.creationTime return taskResult
self._log.exception("RabbitMQ connection failed") raise except amqp.exceptions.AmqpChannelError: self._log.exception("RabbitMQ channel failed") raise except Exception as ex: self._log.exception("Error Streaming data: %s", ex) raise except KeyboardInterrupt: self._log.info("Stopping grok Notification Service: KeyboardInterrupt") finally: self._log.info("Grok Notification Service is exiting") if __name__ == "__main__": g_log = getExtendedLogger("NotificationService") logging_support.LoggingSupport.initService() try: NotificationService().run() except KeyboardInterrupt: pass except Exception as e: outp = StringIO.StringIO() ex_type, ex, tb = sys.exc_info() traceback.print_tb(tb, file=outp) outp.seek(0) g_log.info(outp.read()) g_log.exception("Unexpected Error") sys.exit(1)
import grok.app import grok.app.adapters.datasource as datasource_adapter_factory from grok.app.adapters.datasource.cloudwatch import aws_base import grok.app.exceptions as app_exceptions from grok.app import repository from grok.app.repository import schema from htmengine.repository.queries import MetricStatus import htmengine.utils from grok.test_utils.app.test_case_base import TestCaseBase, unittest import grok.test_utils.aws_utils _LOG = getExtendedLogger("cw_datasource_adapter_base_test") class CloudwatchDatasourceAdapterBaseTest(TestCaseBase): # EC2 Region name of the test ec2 instance _testRegion = None _testId = None _modelSpecNoMinMax = None _modelSpecWithMinMax = None # What we expect to be supported so far _expectedResourceTypes = [ aws_base.ResourceTypeNames.EC2_INSTANCE,
def _getLogger(): return getExtendedLogger(_MODULE_NAME)
metrics_api, models_api, notifications_api, settings_api, support_api, update_api, wufoo_api, UnauthorizedResponse) from grok.app.webservices.utils import encodeJson from htmengine.utils import jsonDecode logging_support.LoggingSupport.initService() log = getExtendedLogger("webapp-logger") urls = ( # Web UI "", "DefaultHandler", "/", "DefaultHandler", "/grok", "GrokHandler", r"/grok/([-\/\w]*)", "GrokHandler", # i18n strings "/_msgs", "MessagesHandler", # REST API Endpoints "/_annotations", annotations_api.app,
from grok.grok_logging import getExtendedLogger import grok.app import grok.app.adapters.datasource as datasource_adapter_factory from grok.app.adapters.datasource.cloudwatch import aws_base from grok.app import repository from grok.app.repository.queries import MetricStatus import htmengine.utils from htmengine.utils import jsonDecode from grok.test_utils.app.test_case_base import TestCaseBase, unittest import grok.test_utils.aws_utils _LOG = getExtendedLogger("cw_datasource_adapter_metrics_test") def setUpModule(): logging_support.LoggingSupport.initTestApp() class CloudwatchDatasourceAdapterMetricsTest(TestCaseBase): _supportedResourceTypes = [ aws_base.ResourceTypeNames.EC2_INSTANCE, aws_base.ResourceTypeNames.AUTOSCALING_GROUP, #aws_base.ResourceTypeNames.DYNAMODB_TABLE, (NO DATA)
from grok.app.webservices import (AuthenticatedBaseHandler, ManagedConnectionWebapp) from grok.app.webservices.models_api import ModelHandler from grok.app.webservices.responses import ( quotaErrorResponseWrapper, InvalidRequestResponse) from grok.app.webservices.utils import encodeJson, loadSchema # Number of instances to suggest (pre-checked) _NUM_SUGGESTED_INSTANCES = 8 # Max number of instances to suggest including alternates (unchecked) _MAX_SUGGESTED_INSTANCES_TOTAL = 28 # Time limit for fetching AWS instances _AWS_INSTANCE_FETCHING_TIME_LIMIT = 5.0 log = grok_logging.getExtendedLogger("webservices") urls = ( "/(.+?)/(.+?\/.+?)/(.+\/*.+)", "InstanceDefaultsHandler", "/(.+)/(.+\/.+)", "InstanceDefaultsHandler", "/?", "InstancesHandler", # /_instances/suggestions "/suggestions", "InstanceSuggestionsHandler", # /_instances/suggestions/us-west-2 "/suggestions/(.+?)", "InstanceSuggestionsHandler", ) # Validation schema to ensure we are getting an array of strings. # Does not validate valid region or namespace, those are handled elsewhere. _INSTANCES_MODEL_CREATION_SCHEMA = loadSchema( "instances_model_creation_schema.json")
from grok import logging_support from grok.grok_logging import getExtendedLogger import grok.app import grok.app.adapters.datasource as datasource_adapter_factory from grok.app.adapters.datasource.cloudwatch import aws_base import grok.app.exceptions as app_exceptions from grok.app import repository from grok.app.repository import schema from htmengine.repository.queries import MetricStatus import htmengine.utils from grok.test_utils.app.test_case_base import TestCaseBase, unittest import grok.test_utils.aws_utils _LOG = getExtendedLogger("cw_datasource_adapter_base_test") class CloudwatchDatasourceAdapterBaseTest(TestCaseBase): # EC2 Region name of the test ec2 instance _testRegion = None _testId = None _modelSpecNoMinMax = None _modelSpecWithMinMax = None # What we expect to be supported so far _expectedResourceTypes = [ aws_base.ResourceTypeNames.EC2_INSTANCE, aws_base.ResourceTypeNames.AUTOSCALING_GROUP,