def test_init_kubejobs(self, mock_config):
"""
Test the KubeJobs Plugin constructor, checking if the
plugin's attributes are equals that atributes given
Args: (patch) -- Path of the Mock of Kubernetes config
Returns: None
"""
mock_config.return_value = None
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
self.assertEqual(plugin.app_id, self.app_id)
self.assertEqual(plugin.info_plugin, self.info_plugin)
self.assertEqual(plugin.collect_period, self.collect_period)
self.assertEqual(plugin.attempts, self.retries)
plugin2 = KubeJobProgress(self.app_id, self.info_plugin)
self.assertEqual(plugin2.app_id, self.app_id)
self.assertEqual(plugin2.info_plugin, self.info_plugin)
self.assertEqual(plugin2.collect_period, 2)
self.assertEqual(plugin2.attempts, 10)
self.assertFalse(plugin == plugin2)
def test_send_monasca_metrics(self):
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.rds = MockRedis()
plugin.b_v1 = MockKube(plugin.app_id)
plugin.monasca = MockMonascaConnector()
plugin.enable_monasca = True
plugin._publish_measurement(5000)
self.assertEqual(len(plugin.monasca.metrics['time-progress']), 1)
self.assertEqual(len(plugin.monasca.metrics['job-progress']), 1)
self.assertEqual(
len(plugin.monasca.metrics['application-progress.error']), 1)
self.assertEqual(len(plugin.monasca.metrics['job-parallelism']), 1)
plugin._publish_measurement(1000)
self.assertEqual(len(plugin.monasca.metrics['time-progress']), 2)
self.assertEqual(len(plugin.monasca.metrics['job-progress']), 2)
self.assertEqual(
len(plugin.monasca.metrics['application-progress.error']), 2)
self.assertEqual(len(plugin.monasca.metrics['job-parallelism']), 2)
def test_get_elapsed_time(self):
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
datetime_now = datetime.now()
elapsed_time = datetime_now - plugin.submission_time
self.assertEqual(elapsed_time.seconds, plugin._get_elapsed_time())
def test_send_monasca_metrics(self, mock_config):
"""
Verify that when the flag enable_monasca is True,
the metrics are delivered to Monasca
Args: (patch) -- Path of the Mock of Kubernetes config
Returns: None
"""
mock_config.return_value = None
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.rds = MockRedis()
plugin.b_v1 = MockKube(plugin.app_id)
plugin.datasource = MockMonascaConnector()
plugin._publish_measurement(5000)
self.assertEqual(len(plugin.datasource.metrics['time-progress']), 1)
self.assertEqual(len(plugin.datasource.metrics['job-progress']), 1)
self.assertEqual(
len(plugin.datasource.metrics['application-progress.error']), 1)
self.assertEqual(len(plugin.datasource.metrics['job-parallelism']), 1)
plugin._publish_measurement(1000)
self.assertEqual(len(plugin.datasource.metrics['time-progress']), 2)
self.assertEqual(len(plugin.datasource.metrics['job-progress']), 2)
self.assertEqual(
len(plugin.datasource.metrics['application-progress.error']), 2)
self.assertEqual(len(plugin.datasource.metrics['job-parallelism']), 2)
def get_monitor(self, plugin, app_id, plugin_info):
executor = None
if plugin == "spark_sahara":
executor = SparkProgress(app_id, plugin_info)
elif plugin == "web_app":
executor = WebAppMonitor(app_id, plugin_info, api.os_keypair)
elif plugin == "openstack_generic":
executor = OSGeneric(app_id, plugin_info, api.os_keypair,
api.retries)
elif plugin == "spark_mesos":
executor = SparkProgressUPV(app_id,
plugin_info,
retries=api.retries)
elif plugin == "kubejobs":
executor = KubeJobProgress(app_id,
plugin_info,
retries=api.retries)
else:
raise ex.BadRequestException()
return executor
def get_monitor(self, plugin, app_id, plugin_info):
executor = None
if plugin == "spark_sahara":
executor = SparkProgress(app_id, plugin_info)
elif plugin == "web_app":
executor = WebAppMonitor(app_id, plugin_info, api.os_keypair)
elif plugin == "openstack_generic":
executor = OSGeneric(app_id, plugin_info, api.os_keypair,
api.retries)
elif plugin == "spark_mesos":
executor = SparkProgressUPV(app_id, plugin_info, retries=api.retries)
elif plugin == "kubejobs":
executor = KubeJobProgress(app_id, plugin_info, retries=api.retries)
elif plugin == "external_api":
plugin_info['threshold'] = api.threshold
plugin_info['metric_source'] = api.metric_source
plugin_info['get_metric_endpoint'] = api.get_metric_endpoint
plugin_info['k8s_manifest'] = api.k8s_manifest
executor = VerticalProgress(app_id, plugin_info, retries=api.retries)
else:
raise ex.BadRequestException()
return executor
def test_get_elapsed_time(self, mock_config):
"""
Check that the elapsed time returned (in seconds)
is equal that elapsed time given
Args: (patch) -- Path of the Mock of Kubernetes config
Returns: None
"""
mock_config.return_value = None
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
datetime_now = datetime.now()
elapsed_time = datetime_now - plugin.submission_time
self.assertEqual(elapsed_time.seconds, plugin._get_elapsed_time())
def test_init_kubejobs(self):
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
self.assertEqual(plugin.app_id, self.app_id)
self.assertEqual(plugin.info_plugin, self.info_plugin)
self.assertEqual(plugin.collect_period, self.collect_period)
self.assertEqual(plugin.attempts, self.retries)
plugin2 = KubeJobProgress(self.app_id, self.info_plugin)
self.assertEqual(plugin2.app_id, self.app_id)
self.assertEqual(plugin2.info_plugin, self.info_plugin)
self.assertEqual(plugin2.collect_period, 2)
self.assertEqual(plugin2.attempts, 10)
self.assertFalse(plugin == plugin2)
def test_monitoring_application(self):
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.rds = MockRedis()
plugin.b_v1 = MockKube(plugin.app_id)
with requests_mock.Mocker() as m:
m.get('http://%s/redis-%s/job/count' %
(plugin.submission_url, plugin.app_id),
text='500')
m.get('http://%s/redis-%s/job:processing/count' %
(plugin.submission_url, plugin.app_id),
text='750')
self.assertEqual(plugin.monitoring_application(), 250)
def test_monitoring_application(self, mock_config):
"""
Check that the function monitoring_application
returns the number of jobs minus the number of
jobs that are be processing plus number of jobs to do
Args: (patch) -- Path of the Mock of Kubernetes config
Returns: None
"""
mock_config.return_value = None
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.rds = MockRedis()
plugin.b_v1 = MockKube(plugin.app_id)
plugin.datasource = MockInfluxConnector()
with requests_mock.Mocker() as m:
m.get('http://%s/redis-%s/job/count' % (plugin.submission_url,
plugin.app_id), text='500')
m.get('http://%s/redis-%s/job:processing/count' % (plugin.submission_url,
plugin.app_id), text='750')
self.assertEqual(plugin.monitoring_application(), 250)
def test_publish_measurement(self):
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.rds = MockRedis()
plugin.b_v1 = MockKube(plugin.app_id)
plugin._publish_measurement(500)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) != None)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) == None)
plugin._publish_measurement(600)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) != None)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) == None)
def test_get_num_replicas(self):
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.b_v1 = MockKube(plugin.app_id)
self.assertEqual(plugin._get_num_replicas(), 1)
plugin.b_v1 = MockKube(plugin.app_id, 3)
self.assertEqual(plugin._get_num_replicas(), 3)
def test_publish_measurement(self, mock_config):
"""
Verify that when a measurement is pubished, the
metrics are delivered to Redis queue
Args: (patch) -- Path of the Mock of Kubernetes config
Returns: None
"""
mock_config.return_value = None
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.rds = MockRedis()
plugin.b_v1 = MockKube(plugin.app_id)
plugin.datasource = MockInfluxConnector()
plugin._publish_measurement(500)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) is not None)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) is None)
plugin._publish_measurement(600)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) is not None)
self.assertTrue(plugin.rds.rpop(plugin.metric_queue) is None)
def test_get_num_replicas(self, mock_config):
"""
Verify that the number of replicas returned are equal
that number of replicas initial
Args: (patch) -- Path of the Mock of Kubernetes config
Returns: None
"""
mock_config.return_value = None
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.b_v1 = MockKube(plugin.app_id)
self.assertEqual(plugin._get_num_replicas(), 1)
plugin.b_v1 = MockKube(plugin.app_id, 3)
self.assertEqual(plugin._get_num_replicas(), 3)
def test_monitoring_application(self, mock_config):
"""
Check that the function monitoring_application
returns the number of jobs minus the number of
jobs that are be processing plus number of jobs to do
Args: (patch) -- Path of the Mock of Kubernetes config
Returns: None
"""
mock_config.return_value = None
plugin = KubeJobProgress(self.app_id, self.info_plugin,
self.collect_period, self.retries)
plugin.rds = MockRedis()
plugin.b_v1 = MockKube(plugin.app_id)
plugin.datasource = MockInfluxConnector()
for i in range(5):
plugin.rds.rpush('job', 'job')
for i in range(10):
plugin.rds.rpush('job:processing', 'job')
self.assertEqual(plugin.monitoring_application(), 1485)