def main_service(): """Method to use with Openstack service.""" ConfigInitializer.basic_config() LogUtils.init_logger(__name__) launcher = os_service.ServiceLauncher(cfg.CONF, restart_method='mutate') launcher.launch_service(Transform()) launcher.wait()
def test_use_default_config_file(self): ConfigInitializer.basic_config(default_config_files=[]) self.assertEqual( 'monasca_transform.mysql_offset_specs:MySQLOffsetSpecs', cfg.CONF.repositories.offsets) self.assertEqual( 'monasca_transform.data_driven_specs.' 'mysql_data_driven_specs_repo:MySQLDataDrivenSpecsRepo', cfg.CONF.repositories.data_driven_specs) self.assertEqual('mysql:thin', cfg.CONF.database.server_type) self.assertEqual('localhost', cfg.CONF.database.host) self.assertEqual('monasca_transform', cfg.CONF.database.database_name) self.assertEqual('m-transform', cfg.CONF.database.username) self.assertEqual('password', cfg.CONF.database.password) self.assertIsNone(cfg.CONF.database.ca_file) self.assertFalse(cfg.CONF.database.use_ssl)
def invoke(): # object to keep track of offsets ConfigInitializer.basic_config() # app name application_name = "mon_metrics_kafka" my_spark_conf = SparkConf().setAppName(application_name) spark_context = SparkContext(conf=my_spark_conf) # read at the configured interval spark_streaming_context = \ StreamingContext(spark_context, cfg.CONF.service.stream_interval) kafka_stream = MonMetricsKafkaProcessor.get_kafka_stream( cfg.CONF.messaging.topic, spark_streaming_context) # transform to recordstore MonMetricsKafkaProcessor.transform_to_recordstore(kafka_stream) # catch interrupt, stop streaming context gracefully # signal.signal(signal.SIGINT, signal_handler) # start processing spark_streaming_context.start() # FIXME: stop spark context to relinquish resources # FIXME: specify cores, so as not to use all the resources on the cluster. # FIXME: HA deploy multiple masters, may be one on each control node try: # Wait for the Spark driver to "finish" spark_streaming_context.awaitTermination() except Exception as e: MonMetricsKafkaProcessor.log_debug( "Exception raised during Spark execution : " + str(e)) # One exception that can occur here is the result of the saved # kafka offsets being obsolete/out of range. Delete the saved # offsets to improve the chance of success on the next execution. # TODO(someone) prevent deleting all offsets for an application, # but just the latest revision MonMetricsKafkaProcessor.log_debug( "Deleting saved offsets for chance of success on next execution") MonMetricsKafkaProcessor.reset_kafka_offsets(application_name) # delete pre hourly processor offsets if cfg.CONF.stage_processors.pre_hourly_processor_enabled: PreHourlyProcessor.reset_kafka_offsets()
def setUp(self): super(SparkTest, self).setUp() # configure the system with a dummy messaging adapter ConfigInitializer.basic_config( default_config_files=[ 'tests/unit/test_resources/config/' 'test_config_with_dummy_messaging_adapter.conf']) # reset metric_id list dummy adapter if not DummyAdapter.adapter_impl: DummyAdapter.init() DummyAdapter.adapter_impl.metric_list = []
def setUp(self): super(TestVmCpuAllocatedAgg, self).setUp() # configure the system with a dummy messaging adapter ConfigInitializer.basic_config(default_config_files=[ 'tests/functional/test_resources/config/' 'test_config_with_dummy_messaging_adapter.conf' ]) # reset metric_id list dummy adapter if not DummyAdapter.adapter_impl: DummyAdapter.init() DummyAdapter.adapter_impl.metric_list = []
def setUp(self): super(TestFetchQuantityInstanceUsageAgg, self).setUp() # configure the system with a dummy messaging adapter ConfigInitializer.basic_config( default_config_files=[ 'tests/functional/test_resources/config/' 'test_config_with_dummy_messaging_adapter.conf']) # reset metric_id list dummy adapter if not DummyAdapter.adapter_impl: DummyAdapter.init() DummyAdapter.adapter_impl.metric_list = []
def invoke(): # object to keep track of offsets ConfigInitializer.basic_config() # app name application_name = "mon_metrics_kafka" my_spark_conf = SparkConf().setAppName(application_name) spark_context = SparkContext(conf=my_spark_conf) # read at the configured interval spark_streaming_context = \ StreamingContext(spark_context, cfg.CONF.service.stream_interval) kafka_stream = MonMetricsKafkaProcessor.get_kafka_stream( cfg.CONF.messaging.topic, spark_streaming_context) # transform to recordstore MonMetricsKafkaProcessor.transform_to_recordstore(kafka_stream) # catch interrupt, stop streaming context gracefully # signal.signal(signal.SIGINT, signal_handler) # start processing spark_streaming_context.start() # FIXME: stop spark context to relinquish resources # FIXME: specify cores, so as not to use all the resources on the cluster. # FIXME: HA deploy multiple masters, may be one on each control node try: # Wait for the Spark driver to "finish" spark_streaming_context.awaitTermination() except Exception as e: MonMetricsKafkaProcessor.log_debug( "Exception raised during Spark execution : " + str(e)) # One exception that can occur here is the result of the saved # kafka offsets being obsolete/out of range. Delete the saved # offsets to improve the chance of success on the next execution. # TODO(someone) prevent deleting all offsets for an application, # but just the latest revision MonMetricsKafkaProcessor.log_debug( "Deleting saved offsets for chance of success on next execution") MonMetricsKafkaProcessor.reset_kafka_offsets(application_name) # delete pre hourly processor offsets if cfg.CONF.stage_processors.pre_hourly_processor_enabled: PreHourlyProcessor.reset_kafka_offsets()
def setUp(self): ConfigInitializer.basic_config(default_config_files=[ 'tests/functional/test_resources/config/test_config.conf' ]) self.config = Config() self.config.config(group='database', use_ssl=True, host='test_ssl_hostname', server_type='jdbc_driver', database_name='db_name', username='******', password='******', ca_file='ca_file') self.config.setUp()
def test_use_specific_config_file(self): ConfigInitializer.basic_config(default_config_files=[ 'tests/functional/test_resources/config/test_config.conf' ]) self.assertEqual('test_offsets_repo_class', cfg.CONF.repositories.offsets) self.assertEqual('test_data_driven_specs_repo_class', cfg.CONF.repositories.data_driven_specs) self.assertEqual('test_server_type', cfg.CONF.database.server_type) self.assertEqual('test_host_name', cfg.CONF.database.host) self.assertEqual('test_database_name', cfg.CONF.database.database_name) self.assertEqual('test_database_user_name', cfg.CONF.database.username) self.assertEqual('test_database_password', cfg.CONF.database.password) self.assertEqual('test_ca_file_path', cfg.CONF.database.ca_file) self.assertTrue(cfg.CONF.database.use_ssl)
def setUp(self): ConfigInitializer.basic_config( default_config_files=[ 'tests/functional/test_resources/config/test_config.conf' ]) self.config = Config() self.config.config( group='database', use_ssl=True, host='test_ssl_hostname', server_type='jdbc_driver', database_name='db_name', username='******', password='******', ca_file='ca_file') self.config.setUp()
def insert(transform_context, instance_usage_df): """write instance usage data to kafka""" # object to init config ConfigInitializer.basic_config() transform_spec_df = transform_context.transform_spec_df_info agg_params = transform_spec_df.select( "aggregation_params_map.dimension_list").collect()[0].asDict() # Approach # 1 # using foreachPartition to iterate through elements in an # RDD is the recommended approach so as to not overwhelm kafka with the # zillion connections (but in our case the MessageAdapter does # store the adapter_impl so we should not create many producers) # using foreachpartitions was causing some serialization/cpickle # problems where few libs like kafka.SimpleProducer and oslo_config.cfg # were not available in foreachPartition method # # removing _write_metrics_from_partition for now in favor of # Approach # 2 # # instance_usage_df_agg_params = instance_usage_df.rdd.map( # lambda x: InstanceUsageDataAggParams(x, # agg_params)) # instance_usage_df_agg_params.foreachPartition( # DummyInsert._write_metrics_from_partition) # Approach # 2 # using collect() to fetch all elements of an RDD and write to # kafka for instance_usage_row in instance_usage_df.collect(): metric = InsertComponent._get_metric(instance_usage_row, agg_params) # validate metric part if InsertComponent._validate_metric(metric): KafkaMessageAdapter.send_metric(metric) return instance_usage_df
def insert(transform_context, instance_usage_df): """write instance usage data to kafka""" # object to init config ConfigInitializer.basic_config() transform_spec_df = transform_context.transform_spec_df_info agg_params = transform_spec_df.select( "aggregation_params_map.dimension_list").collect()[0].asDict() # Approach # 1 # using foreachPartition to iterate through elements in an # RDD is the recommended approach so as to not overwhelm kafka with the # zillion connections (but in our case the MessageAdapter does # store the adapter_impl so we should not create many producers) # using foreachpartitions was causing some serialization/cpickle # problems where few libs like kafka.SimpleProducer and oslo_config.cfg # were not available in foreachPartition method # # removing _write_metrics_from_partition for now in favor of # Approach # 2 # # instance_usage_df_agg_params = instance_usage_df.rdd.map( # lambda x: InstanceUsageDataAggParams(x, # agg_params)) # instance_usage_df_agg_params.foreachPartition( # DummyInsert._write_metrics_from_partition) # Approach # 2 # using collect() to fetch all elements of an RDD and write to # kafka for instance_usage_row in instance_usage_df.collect(): metric = InsertComponent._get_metric( instance_usage_row, agg_params) # validate metric part if InsertComponent._validate_metric(metric): KafkaMessageAdapter.send_metric(metric) return instance_usage_df
def test_use_default_config_file(self): ConfigInitializer.basic_config(default_config_files=[]) self.assertEqual( 'monasca_transform.mysql_offset_specs:MySQLOffsetSpecs', cfg.CONF.repositories.offsets) self.assertEqual( 'monasca_transform.data_driven_specs.' 'mysql_data_driven_specs_repo:MySQLDataDrivenSpecsRepo', cfg.CONF.repositories.data_driven_specs) self.assertEqual('mysql:thin', cfg.CONF.database.server_type) self.assertEqual('localhost', cfg.CONF.database.host) self.assertEqual('monasca_transform', cfg.CONF.database.database_name) self.assertEqual('m-transform', cfg.CONF.database.username) self.assertEqual('password', cfg.CONF.database.password) self.assertIsNone(cfg.CONF.database.ca_file) self.assertFalse(cfg.CONF.database.use_ssl)
def test_use_specific_config_file(self): ConfigInitializer.basic_config( default_config_files=[ 'tests/unit/test_resources/config/test_config.conf' ]) self.assertEqual('test_offsets_repo_class', cfg.CONF.repositories.offsets) self.assertEqual('test_data_driven_specs_repo_class', cfg.CONF.repositories.data_driven_specs) self.assertEqual('test_server_type', cfg.CONF.database.server_type) self.assertEqual('test_host_name', cfg.CONF.database.host) self.assertEqual('test_database_name', cfg.CONF.database.database_name) self.assertEqual('test_database_user_name', cfg.CONF.database.username) self.assertEqual('test_database_password', cfg.CONF.database.password)
def insert(transform_context, instance_usage_df): """write instance usage data to kafka""" # object to init config ConfigInitializer.basic_config() transform_spec_df = transform_context.transform_spec_df_info agg_params = transform_spec_df.select( "metric_id").\ collect()[0].asDict() metric_id = agg_params["metric_id"] for instance_usage_row in instance_usage_df.collect(): instance_usage_dict = \ InsertComponent._get_instance_usage_pre_hourly( instance_usage_row, metric_id) KafkaMessageAdapterPreHourly.send_metric(instance_usage_dict) return instance_usage_df
def insert(transform_context, instance_usage_df): """write instance usage data to kafka""" # object to init config ConfigInitializer.basic_config() transform_spec_df = transform_context.transform_spec_df_info agg_params = transform_spec_df.select( "metric_id").\ collect()[0].asDict() metric_id = agg_params["metric_id"] for instance_usage_row in instance_usage_df.collect(): instance_usage_dict = \ InsertComponent._get_instance_usage_pre_hourly( instance_usage_row, metric_id) KafkaMessageAdapterPreHourly.send_metric(instance_usage_dict) return instance_usage_df
def setUp(self): ConfigInitializer.basic_config() super(TestMySQLDataDrivenSpecsRepo, self).setUp() self.data_driven_specs_repo = MySQLDataDrivenSpecsRepo()
def setUp(self): super(TransformBuilderTest, self).setUp() # configure the system with a dummy messaging adapter ConfigInitializer.basic_config( default_config_files=[ 'tests/functional/test_resources/config/test_config.conf'])
def setUp(self): ConfigInitializer.basic_config() super(TestMySQLDataDrivenSpecsRepo, self).setUp() self.data_driven_specs_repo = MySQLDataDrivenSpecsRepo()
def setUp(self): ConfigInitializer.basic_config() self.kafka_offset_specs = MySQLOffsetSpecs()
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import datetime import unittest from monasca_transform.config.config_initializer import ConfigInitializer ConfigInitializer.basic_config( default_config_files=[ 'tests/unit/test_resources/config/' 'test_config.conf'] ) from monasca_transform.processor.processor_util import PreHourlyProcessorUtil from monasca_transform.processor.processor_util import ProcessUtilDataProvider class PreHourlyProcessorTest(unittest.TestCase): def setUp(self): pass def test_is_time_to_run_before_late_metric_slack_time(self): check_time = datetime.datetime( year=2016, month=11, day=7, hour=11, minute=9, second=59, microsecond=0)
from monasca_transform.data_driven_specs.data_driven_specs_repo \ import DataDrivenSpecsRepo from monasca_transform.data_driven_specs.data_driven_specs_repo \ import DataDrivenSpecsRepoFactory from monasca_transform.processor.pre_hourly_processor import PreHourlyProcessor from monasca_transform.transform import RddTransformContext from monasca_transform.transform.storage_utils import \ InvalidCacheStorageLevelException from monasca_transform.transform.storage_utils import StorageUtils from monasca_transform.transform.transform_utils import MonMetricUtils from monasca_transform.transform import TransformContextUtils ConfigInitializer.basic_config() log = LogUtils.init_logger(__name__) class MonMetricsKafkaProcessor(object): @staticmethod def log_debug(message): print(message) log.debug(message) @staticmethod def store_offset_ranges(batch_time, rdd): if rdd.isEmpty(): MonMetricsKafkaProcessor.log_debug( "storeOffsetRanges: nothing to process...")
def setUp(self): super(TransformBuilderTest, self).setUp() # configure the system with a dummy messaging adapter ConfigInitializer.basic_config(default_config_files=[ 'tests/functional/test_resources/config/test_config.conf' ])
# a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import datetime import unittest from monasca_transform.config.config_initializer import ConfigInitializer ConfigInitializer.basic_config(default_config_files=[ 'tests/unit/test_resources/config/' 'test_config.conf' ]) from monasca_transform.processor.processor_util import PreHourlyProcessorUtil from monasca_transform.processor.processor_util import ProcessUtilDataProvider class PreHourlyProcessorTest(unittest.TestCase): def setUp(self): pass def test_is_time_to_run_before_late_metric_slack_time(self): check_time = datetime.datetime(year=2016, month=11, day=7, hour=11, minute=9,
def setUp(self): ConfigInitializer.basic_config() self.kafka_offset_specs = MySQLOffsetSpecs()
import GenericTransformBuilder from monasca_transform.data_driven_specs.data_driven_specs_repo \ import DataDrivenSpecsRepo from monasca_transform.data_driven_specs.data_driven_specs_repo \ import DataDrivenSpecsRepoFactory from monasca_transform.processor.pre_hourly_processor import PreHourlyProcessor from monasca_transform.transform import RddTransformContext from monasca_transform.transform.storage_utils import StorageUtils from monasca_transform.transform.transform_utils import MonMetricUtils from monasca_transform.transform import TransformContextUtils ConfigInitializer.basic_config() # initialize logger log = logging.getLogger(__name__) _h = logging.FileHandler('%s/%s' % ( cfg.CONF.service.service_log_path, cfg.CONF.service.service_log_filename) ) _h.setFormatter(logging.Formatter("'%(asctime)s - %(pathname)s:" "%(lineno)s - %(levelname)s - %(message)s'")) log.addHandler(_h) log.setLevel(logging.DEBUG) class MonMetricsKafkaProcessor(object):