def setUpModule():
logging_support.LoggingSupport.initTestApp()
global g_log # pylint: disable=W0603
g_log = getExtendedLogger("autostack_datasource_adapter_test")
import YOMP.app
import YOMP.app.adapters.datasource as datasource_adapter_factory
from YOMP.app.adapters.datasource.cloudwatch import aws_base
import YOMP.app.exceptions as app_exceptions
from YOMP.app import repository
from YOMP.app.repository import schema
from htmengine.repository.queries import MetricStatus
import htmengine.utils
from YOMP.test_utils.app.test_case_base import TestCaseBase, unittest
import YOMP.test_utils.aws_utils
_LOG = getExtendedLogger("cw_datasource_adapter_base_test")
class CloudwatchDatasourceAdapterBaseTest(TestCaseBase):
# EC2 Region name of the test ec2 instance
_testRegion = None
_testId = None
_modelSpecNoMinMax = None
_modelSpecWithMinMax = None
# What we expect to be supported so far
_expectedResourceTypes = [
aws_base.ResourceTypeNames.EC2_INSTANCE,
from YOMP import logging_support
from YOMP.YOMP_logging import getExtendedLogger
import YOMP.app
import YOMP.app.adapters.datasource as datasource_adapter_factory
from YOMP.app.adapters.datasource.cloudwatch import aws_base
import YOMP.app.exceptions as app_exceptions
from YOMP.app import repository
from YOMP.app.repository import schema
from htmengine.repository.queries import MetricStatus
import htmengine.utils
from YOMP.test_utils.app.test_case_base import TestCaseBase, unittest
import YOMP.test_utils.aws_utils
_LOG = getExtendedLogger("cw_datasource_adapter_base_test")
class CloudwatchDatasourceAdapterBaseTest(TestCaseBase):
# EC2 Region name of the test ec2 instance
_testRegion = None
_testId = None
_modelSpecNoMinMax = None
_modelSpecWithMinMax = None
# What we expect to be supported so far
_expectedResourceTypes = [
aws_base.ResourceTypeNames.EC2_INSTANCE,
aws_base.ResourceTypeNames.AUTOSCALING_GROUP,
from YOMP.YOMP_logging import getExtendedLogger
import YOMP.app
import YOMP.app.adapters.datasource as datasource_adapter_factory
from YOMP.app.adapters.datasource.cloudwatch import aws_base
from YOMP.app import repository
from YOMP.app.repository.queries import MetricStatus
import htmengine.utils
from htmengine.utils import jsonDecode
from YOMP.test_utils.app.test_case_base import TestCaseBase, unittest
import YOMP.test_utils.aws_utils
_LOG = getExtendedLogger("cw_datasource_adapter_metrics_test")
def setUpModule():
logging_support.LoggingSupport.initTestApp()
class CloudwatchDatasourceAdapterMetricsTest(TestCaseBase):
_supportedResourceTypes = [
aws_base.ResourceTypeNames.EC2_INSTANCE,
aws_base.ResourceTypeNames.AUTOSCALING_GROUP,
#aws_base.ResourceTypeNames.DYNAMODB_TABLE, (NO DATA)
from YOMP.app.aws import instance_utils
from YOMP.app.aws.ec2_utils import checkEC2Authorization
from YOMP.app.exceptions import AuthFailure, AWSPermissionsError
from YOMP.app.webservices import (annotations_api, anomalies_api,
autostacks_api, instances_api,
getWebLogPrefix, logging_api, messagemanager,
metrics_api, models_api, notifications_api,
settings_api, support_api, update_api,
wufoo_api, UnauthorizedResponse)
from YOMP.app.webservices.utils import encodeJson
from htmengine.utils import jsonDecode
logging_support.LoggingSupport.initService()
log = getExtendedLogger("webapp-logger")
urls = (
# Web UI
"",
"DefaultHandler",
"/",
"DefaultHandler",
"/YOMP",
"YOMPHandler",
r"/YOMP/([-\/\w]*)",
"YOMPHandler",
# i18n strings
"/_msgs",
"MessagesHandler",
def _getCollectWorkerLogger(): return getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
def _getInstanceWorkerLogger():
return getExtendedLogger(_MODULE_NAME + ".INSTANCE")
def _collectMetrics(task):
""" Executed via multiprocessing Pool: Collect metric data
:param task: a _MetricCollectionTask object
:returns: a _MetricCollectionTaskResult object
"""
log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
taskResult = _MetricCollectionTaskResult(refID=task.refID,
metricID=task.metricID,
instanceID=task.instanceID)
try:
if task.metricName == "InstanceCount":
timestamp = datetime.utcnow().replace(second=0, microsecond=0)
taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0), )
else:
backoffSec = 0.75
backoffGrowthFactor = 1.5
maxBackoffSec = 5
rawdata = None
while rawdata is None:
try:
cw = cloudwatch.connect_to_region(region_name=task.region,
**getAWSCredentials())
rawdata = cw.get_metric_statistics(
period=task.period,
start_time=task.timeRange.start,
end_time=task.timeRange.end,
metric_name=task.metricName,
namespace="AWS/EC2",
statistics=task.stats,
dimensions=dict(InstanceId=task.instanceID),
unit=task.unit)
except BotoServerError as e:
if e.status == 400 and e.error_code == "Throttling":
# TODO: unit-test
log.info("Throttling: %r", e)
if backoffSec <= maxBackoffSec:
time.sleep(backoffSec)
backoffSec *= backoffGrowthFactor
else:
raise app_exceptions.MetricThrottleError(repr(e))
else:
raise
# Sort the data by timestamp in ascending order
rawdata.sort(key=lambda row: row["Timestamp"])
# Convert raw data to MetricRecord objects
def getValue(rawDataItem, stats):
if isinstance(stats, basestring):
return rawDataItem[stats]
else:
return dict((field, rawDataItem[field]) for field in stats)
taskResult.data = tuple(
MetricRecord(timestamp=rawItem["Timestamp"],
value=getValue(rawItem, task.stats))
for rawItem in rawdata)
except Exception as e: # pylint: disable=W0703
log.exception("Error in task=%r", task)
taskResult.exception = e
finally:
taskResult.duration = time.time() - taskResult.creationTime
return taskResult
def _getLogger():
return getExtendedLogger(_MODULE_NAME)
def _getLogger():
return getExtendedLogger(_MODULE_NAME)
def setUpModule():
logging_support.LoggingSupport.initTestApp()
global g_log # pylint: disable=W0603
g_log = getExtendedLogger("autostack_datasource_adapter_test")
def __init__(self):
# Make sure we have the latest version of configuration
YOMP.app.config.loadConfig()
self._log = getExtendedLogger(self.__class__.__name__)
self._modelResultsExchange = YOMP.app.config.get("metric_streamer", "results_exchange_name")
except amqp.exceptions.AmqpConnectionError:
self._log.exception("RabbitMQ connection failed")
raise
except amqp.exceptions.AmqpChannelError:
self._log.exception("RabbitMQ channel failed")
raise
except Exception as ex:
self._log.exception("Error Streaming data: %s", ex)
raise
except KeyboardInterrupt:
self._log.info("Stopping YOMP Notification Service: KeyboardInterrupt")
finally:
self._log.info("YOMP Notification Service is exiting")
if __name__ == "__main__":
g_log = getExtendedLogger("NotificationService")
logging_support.LoggingSupport.initService()
try:
NotificationService().run()
except KeyboardInterrupt:
pass
except Exception as e:
outp = StringIO.StringIO()
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb, file=outp)
outp.seek(0)
g_log.info(outp.read())
g_log.exception("Unexpected Error")
sys.exit(1)
def _getInstanceWorkerLogger(): return getExtendedLogger(_MODULE_NAME + ".INSTANCE")
from YOMP import logging_support
from YOMP.YOMP_logging import getExtendedLogger
import YOMP.app
import YOMP.app.adapters.datasource as datasource_adapter_factory
from YOMP.app.adapters.datasource.cloudwatch import aws_base
from YOMP.app import repository
from YOMP.app.repository.queries import MetricStatus
import htmengine.utils
from htmengine.utils import jsonDecode
from YOMP.test_utils.app.test_case_base import TestCaseBase, unittest
import YOMP.test_utils.aws_utils
_LOG = getExtendedLogger("cw_datasource_adapter_metrics_test")
def setUpModule():
logging_support.LoggingSupport.initTestApp()
class CloudwatchDatasourceAdapterMetricsTest(TestCaseBase):
_supportedResourceTypes = [
aws_base.ResourceTypeNames.EC2_INSTANCE,
aws_base.ResourceTypeNames.AUTOSCALING_GROUP,
#aws_base.ResourceTypeNames.DYNAMODB_TABLE, (NO DATA)
aws_base.ResourceTypeNames.ELB_LOAD_BALANCER,
#aws_base.ResourceTypeNames.EBS_VOLUME, (NO DATA)
aws_base.ResourceTypeNames.OPSWORKS_STACK,
def _getCollectWorkerLogger():
return getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
metrics_api,
models_api,
notifications_api,
settings_api,
support_api,
update_api,
wufoo_api,
UnauthorizedResponse)
from YOMP.app.webservices.utils import encodeJson
from htmengine.utils import jsonDecode
logging_support.LoggingSupport.initService()
log = getExtendedLogger("webapp-logger")
urls = (
# Web UI
"", "DefaultHandler",
"/", "DefaultHandler",
"/YOMP", "YOMPHandler",
r"/YOMP/([-\/\w]*)", "YOMPHandler",
# i18n strings
"/_msgs", "MessagesHandler",
# REST API Endpoints
"/_annotations", annotations_api.app,
def _collectMetrics(task):
""" Executed via multiprocessing Pool: Collect metric data
:param task: a _MetricCollectionTask object
:returns: a _MetricCollectionTaskResult object
"""
log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
taskResult = _MetricCollectionTaskResult(
refID=task.refID,
metricID=task.metricID,
instanceID=task.instanceID)
try:
if task.metricName == "InstanceCount":
timestamp = datetime.utcnow().replace(second=0, microsecond=0)
taskResult.data = (MetricRecord(timestamp=timestamp,
value=1.0),)
else:
backoffSec = 0.75
backoffGrowthFactor = 1.5
maxBackoffSec = 5
rawdata = None
while rawdata is None:
try:
cw = cloudwatch.connect_to_region(region_name=task.region,
**getAWSCredentials())
rawdata = cw.get_metric_statistics(
period=task.period,
start_time=task.timeRange.start,
end_time=task.timeRange.end,
metric_name=task.metricName,
namespace="AWS/EC2",
statistics=task.stats,
dimensions=dict(InstanceId=task.instanceID),
unit=task.unit)
except BotoServerError as e:
if e.status == 400 and e.error_code == "Throttling":
# TODO: unit-test
log.info("Throttling: %r", e)
if backoffSec <= maxBackoffSec:
time.sleep(backoffSec)
backoffSec *= backoffGrowthFactor
else:
raise app_exceptions.MetricThrottleError(repr(e))
else:
raise
# Sort the data by timestamp in ascending order
rawdata.sort(key=lambda row: row["Timestamp"])
# Convert raw data to MetricRecord objects
def getValue(rawDataItem, stats):
if isinstance(stats, basestring):
return rawDataItem[stats]
else:
return dict((field, rawDataItem[field]) for field in stats)
taskResult.data = tuple(
MetricRecord(timestamp=rawItem["Timestamp"],
value=getValue(rawItem, task.stats))
for rawItem in rawdata)
except Exception as e: # pylint: disable=W0703
log.exception("Error in task=%r", task)
taskResult.exception = e
finally:
taskResult.duration = time.time() - taskResult.creationTime
return taskResult
