def device_send(self, job_template, job_input, is_delete, retry): config_str = json.dumps(job_input, sort_keys=True) self.push_config_state = PushConfigState.PUSH_STATE_IN_PROGRESS start_time = None config_size = 0 try: config_size = len(config_str) current_config_hash = md5(config_str).hexdigest() if self.last_config_hash is None or\ current_config_hash != self.last_config_hash: self._logger.info("Config push for %s(%s) using job template %s" % (self.physical_router.name, self.physical_router.uuid, str(job_template))) self._logger.debug("Abstract config: %s" % json.dumps(job_input, indent=4, sort_keys=True)) device_manager = DeviceManager.get_instance() job_handler = JobHandler(job_template, job_input, None if is_delete else [self.physical_router.uuid], device_manager.get_api_server_config(), self._logger, device_manager._amqp_client, device_manager._args) self.commit_stats['total_commits_sent_since_up'] += 1 start_time = time.time() job_handler.push(**device_manager.get_job_status_config()) end_time = time.time() self.commit_stats['commit_status_message'] = 'success' self.commit_stats['last_commit_time'] = \ datetime.datetime.fromtimestamp( end_time).strftime('%Y-%m-%d %H:%M:%S') self.commit_stats['last_commit_duration'] = str( end_time - start_time) self.last_config_hash = current_config_hash else: self._logger.debug("not pushing since no config change" " detected") self.push_config_state = PushConfigState.PUSH_STATE_SUCCESS except Exception as e: self._logger.error("Router %s: %s" % (self.physical_router.management_ip, repr(e))) self._logger.error("Abstract config: %s" % json.dumps(job_input, indent=4, sort_keys=True)) self.commit_stats[ 'commit_status_message'] = 'failed to apply config,\ router response: ' + e.message if start_time is not None: self.commit_stats['last_commit_time'] = \ datetime.datetime.fromtimestamp( start_time).strftime('%Y-%m-%d %H:%M:%S') self.commit_stats['last_commit_duration'] = str( time.time() - start_time) self.push_config_state = PushConfigState.PUSH_STATE_RETRY if retry\ else PushConfigState.PUSH_STATE_FAILED return config_size
def device_send(self, job_template, job_input, is_delete, retry): config_str = json.dumps(job_input, sort_keys=True) self.push_config_state = PushConfigState.PUSH_STATE_IN_PROGRESS start_time = None config_size = 0 try: config_size = len(config_str) current_config_hash = md5(config_str).hexdigest() if self.last_config_hash is None or\ current_config_hash != self.last_config_hash: self._logger.info( "Config push for %s(%s) using job template %s" % (self.physical_router.name, self.physical_router.uuid, str(job_template))) self._logger.debug( "Abstract config: %s" % json.dumps(job_input, indent=4, sort_keys=True)) device_manager = DeviceManager.get_instance() job_handler = JobHandler( job_template, job_input, None if is_delete else [self.physical_router.uuid], device_manager.get_api_server_config(), self._logger, device_manager._amqp_client, device_manager._args) self.commit_stats['total_commits_sent_since_up'] += 1 start_time = time.time() job_handler.push(**device_manager.get_job_status_config()) end_time = time.time() self.commit_stats['commit_status_message'] = 'success' self.commit_stats['last_commit_time'] = \ datetime.datetime.fromtimestamp( end_time).strftime('%Y-%m-%d %H:%M:%S') self.commit_stats['last_commit_duration'] = str(end_time - start_time) self.last_config_hash = current_config_hash else: self._logger.debug("not pushing since no config change" " detected") self.push_config_state = PushConfigState.PUSH_STATE_SUCCESS except Exception as e: self._logger.error("Router %s: %s" % (self.physical_router.management_ip, repr(e))) self._logger.error("Abstract config: %s" % json.dumps(job_input, indent=4, sort_keys=True)) self.commit_stats[ 'commit_status_message'] = 'failed to apply config,\ router response: ' + e.message if start_time is not None: self.commit_stats['last_commit_time'] = \ datetime.datetime.fromtimestamp( start_time).strftime('%Y-%m-%d %H:%M:%S') self.commit_stats['last_commit_duration'] = str(time.time() - start_time) self.push_config_state = PushConfigState.PUSH_STATE_RETRY if retry\ else PushConfigState.PUSH_STATE_FAILED return config_size
def device_send(self, job_template, job_input, retry): config_str = json.dumps(job_input) self.push_config_state = PushConfigState.PUSH_STATE_INIT start_time = None config_size = 0 try: self._logger.debug("playbook send message: %s" % config_str) config_size = len(config_str) device_manager = DeviceManager.get_instance() job_handler = JobHandler(job_template, job_input, [self.physical_router.uuid], device_manager.get_analytics_config(), device_manager.get_vnc(), self._logger) self.commit_stats['total_commits_sent_since_up'] += 1 start_time = time.time() job_handler.push() end_time = time.time() self.commit_stats['commit_status_message'] = 'success' self.commit_stats['last_commit_time'] = \ datetime.datetime.fromtimestamp( end_time).strftime('%Y-%m-%d %H:%M:%S') self.commit_stats['last_commit_duration'] = str( end_time - start_time) self.push_config_state = PushConfigState.PUSH_STATE_SUCCESS except Exception as e: self._logger.error("Router %s: %s" % (self.physical_router.management_ip, e.message)) self.commit_stats[ 'commit_status_message'] = 'failed to apply config,\ router response: ' + e.message if start_time is not None: self.commit_stats['last_commit_time'] = \ datetime.datetime.fromtimestamp( start_time).strftime('%Y-%m-%d %H:%M:%S') self.commit_stats['last_commit_duration'] = str( time.time() - start_time) self.push_config_state = PushConfigState.PUSH_STATE_RETRY if retry\ else PushConfigState.PUSH_STATE_FAILED return config_size
class TestJobHandler(unittest.TestCase): NO_LOG_RESPONSE = """ { "value": [] } """ JOB_LOG_RESPONSE = """ { "value": [ { "MessageTS": 12345, "MessageType": "JobLog" } ] } """ TIMEOUT = 15 def setUp(self): super(TestJobHandler, self).setUp() self.post_patch = mock.patch( 'opserver_util.OpServerUtils.post_url_http') self.sleep_patch = mock.patch('gevent.sleep') self.post_mock = self.post_patch.start() self.sleep_mock = self.sleep_patch.start() self.vnc_api = mock.Mock() self.logger = mock.Mock() self.job_type = ['test-job'] self.job_input = {'key1': 'value1', 'key2': 'value2'} self.device_list = ['device1'] self.analytic_config = {'ip': '1.2.3.4', 'port': '8082'} self.job_handler = JobHandler(self.job_type, self.job_input, self.device_list, self.analytic_config, self.vnc_api, self.logger) # end setUp def tearDown(self): super(TestJobHandler, self).tearDown() self.post_patch.stop() self.sleep_patch.stop() # end tearDown def test_job_executed_successfully(self): self.assertFalse(self.job_handler.is_job_done()) self.vnc_api.execute_job.return_value = {'job_execution_id': 'job-1'} self.post_mock.return_value = self.JOB_LOG_RESPONSE self.assertEqual(self.job_handler.get_job_status(), JobStatus.INIT) self.job_handler.push() self.vnc_api.execute_job.assert_called_with( job_template_fq_name=self.job_type, job_input=self.job_input, device_list=self.device_list) self.assertEqual(self.post_mock.call_args[0][1], 'http://1.2.3.4:8082/analytics/query') self.assertEqual(self.post_mock.call_count, 1) self.assertTrue(self.job_handler.is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.COMPLETE) self.assertEqual(self.sleep_mock.call_args_list, []) # end test_job_executed_successfully def test_job_stays_in_progress_then_completes(self): return_values = [ self.NO_LOG_RESPONSE, self.NO_LOG_RESPONSE, self.JOB_LOG_RESPONSE ] def side_effect(*_): self.assertFalse(self.job_handler.is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.IN_PROGRESS) side_effect.counter += 1 return return_values[side_effect.counter - 1] # end side_effect side_effect.counter = 0 self.assertFalse(self.job_handler.is_job_done()) self.vnc_api.execute_job.return_value = {'job_execution_id': 'job-1'} self.post_mock.side_effect = side_effect self.assertEqual(self.job_handler.get_job_status(), JobStatus.INIT) self.job_handler.push() self.vnc_api.execute_job.assert_called_with( job_template_fq_name=self.job_type, job_input=self.job_input, device_list=self.device_list) self.assertEqual(self.post_mock.call_count, 3) self.assertTrue(self.job_handler.is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.COMPLETE) self.assertEqual(self.sleep_mock.call_args_list, [mock.call(self.TIMEOUT)]) # end test_job_stays_in_progress_then_completes def test_job_failed(self): self.assertFalse(self.job_handler.is_job_done()) self.vnc_api.execute_job.return_value = {'job_execution_id': 'job-1'} self.post_mock.side_effect = [ self.NO_LOG_RESPONSE, self.JOB_LOG_RESPONSE ] self.assertEqual(self.job_handler.get_job_status(), JobStatus.INIT) self.assertRaises(Exception, self.job_handler.push) self.vnc_api.execute_job.assert_called_with( job_template_fq_name=self.job_type, job_input=self.job_input, device_list=self.device_list) self.assertEqual(self.post_mock.call_count, 2) self.assertTrue(self.job_handler.is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.FAILED) self.assertEqual(self.sleep_mock.call_args_list, []) # end test_job_failed def test_execute_job_throws(self): self.assertFalse(self.job_handler.is_job_done()) self.vnc_api.execute_job.side_effect = \ cfgm_common.exceptions.HttpError(500, "execute-job failed") self.assertEqual(self.job_handler.get_job_status(), JobStatus.INIT) self.assertRaises(Exception, self.job_handler.push) self.vnc_api.execute_job.assert_called_with( job_template_fq_name=self.job_type, job_input=self.job_input, device_list=self.device_list) self.post_mock.assert_not_called() self.assertTrue(self.job_handler.is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.FAILED) self.sleep_mock.assert_not_called() # end test_execute_job_throws def test_check_status_throws(self): self.assertFalse(self.job_handler.is_job_done()) self.vnc_api.execute_job.return_value = {'job_execution_id': 'job-1'} self.post_mock.side_effect = requests.exceptions.HTTPError() self.assertEqual(self.job_handler.get_job_status(), JobStatus.INIT) self.assertRaises(Exception, self.job_handler.push) self.vnc_api.execute_job.assert_called_with( job_template_fq_name=self.job_type, job_input=self.job_input, device_list=self.device_list) self.assertEqual(self.post_mock.call_count, 1) self.assertTrue(self.job_handler.is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.FAILED) self.sleep_mock.assert_not_called() # end test_check_status_throws def test_max_retries_done(self): self.assertFalse(self.job_handler.is_job_done()) self.vnc_api.execute_job.return_value = {'job_execution_id': 'job-1'} self.post_mock.side_effect = [ self.NO_LOG_RESPONSE, self.NO_LOG_RESPONSE, self.NO_LOG_RESPONSE, self.NO_LOG_RESPONSE, self.NO_LOG_RESPONSE, self.NO_LOG_RESPONSE ] self.assertEqual(self.job_handler.get_job_status(), JobStatus.INIT) self.assertRaises(Exception, self.job_handler.push, self.TIMEOUT, 3) self.vnc_api.execute_job.assert_called_with( job_template_fq_name=self.job_type, job_input=self.job_input, device_list=self.device_list) self.assertEqual(self.post_mock.call_count, 6) self.assertTrue(self.job_handler.is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.FAILED) self.assertEqual(self.sleep_mock.call_args_list, [mock.call(self.TIMEOUT), mock.call(self.TIMEOUT)])
class TestJobHandler(unittest.TestCase): JOB_SUCCESS = AttrDict({ "job_status": "SUCCESS" }) JOB_FAILURE = AttrDict({ "job_status": "FAILURE" }) JOB_IN_PROGRESS = AttrDict({ "job_status": "IN_PROGRESS" }) TIMEOUT = 15 MAX_RETRIES = 30 def setUp(self): super(TestJobHandler, self).setUp() self.sleep_patch = mock.patch('gevent.sleep') self.sleep_mock = self.sleep_patch.start() self.logger = mock.Mock() self.amqp_client = mock.Mock() self.message = mock.Mock() self.job_template_name = ['test-job'] self.job_input = {'key1': 'value1', 'key2': 'value2'} self.device_list = ['device1'] self.api_server_config = { 'ips': ["1.2.3.4"], 'port': "8082", 'username': "******", 'password': "******", 'tenant': "default", 'use_ssl': False } self.args = AttrDict({ 'api_server_ip': '127.0.0.1', 'admin_user': '******', 'admin_password': '******', 'admin_tenant_name': 'test', 'api_server_port': 8082, 'api_server_use_ssl': False }) self.job_handler = JobHandler(self.job_template_name, self.job_input, self.device_list, self.api_server_config, self.logger, self.amqp_client, self.args) # end setUp def tearDown(self): super(TestJobHandler, self).tearDown() self.sleep_patch.stop() # end tearDown def test_job_executed_successfully(self): def side_effect(*_): self.assertFalse(self.job_handler._is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.STARTING) _, kwargs = self.amqp_client.add_consumer.call_args_list[0] callback = kwargs['callback'] callback(self.JOB_SUCCESS, self.message) # end side_effect self.sleep_mock.side_effect = side_effect self.assertFalse(self.job_handler._is_job_done()) self.job_handler.push(self.TIMEOUT, self.MAX_RETRIES) self.amqp_client.add_consumer.assert_called_once() self.amqp_client.publish.assert_called_once() args, kwargs = self.amqp_client.publish.call_args_list[0] job_payload = args[0] job_execution_id = job_payload.get('job_execution_id') self.assertEqual(args[1], JobHandler.JOB_REQUEST_EXCHANGE) self.assertEqual(kwargs['routing_key'], JobHandler.JOB_REQUEST_ROUTING_KEY) args, kwargs = self.amqp_client.add_consumer.call_args_list[0] self.assertEqual(args[0], JobHandler.JOB_STATUS_CONSUMER + job_execution_id) self.assertEqual(args[1], JobHandler.JOB_STATUS_EXCHANGE) self.assertEqual(kwargs['routing_key'], JobHandler.JOB_STATUS_ROUTING_KEY + job_execution_id) self.assertEqual(kwargs['auto_delete'], True) self.message.ack.assert_called_once() self.amqp_client.remove_consumer.assert_called_once() self.assertTrue(self.job_handler._is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.SUCCESS) # end test_job_executed_successfully def test_job_stays_in_progress_then_completes(self): job_statuses = [self.JOB_IN_PROGRESS, self.JOB_IN_PROGRESS, self.JOB_SUCCESS] def side_effect(*_): side_effect.counter += 1 _, kwargs = self.amqp_client.add_consumer.call_args_list[0] callback = kwargs['callback'] callback(job_statuses[side_effect.counter-1], self.message) # end side_effect side_effect.counter = 0 self.sleep_mock.side_effect = side_effect self.assertFalse(self.job_handler._is_job_done()) self.job_handler.push(self.TIMEOUT, self.MAX_RETRIES) self.amqp_client.publish.assert_called_once() self.assertEqual(side_effect.counter, 3) self.assertTrue(self.job_handler._is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.SUCCESS) # end test_job_stays_in_progress_then_completes def test_job_failed(self): job_statuses = [self.JOB_IN_PROGRESS, self.JOB_FAILURE] def side_effect(*_): side_effect.counter += 1 _, kwargs = self.amqp_client.add_consumer.call_args_list[0] callback = kwargs['callback'] callback(job_statuses[side_effect.counter-1], self.message) # end side_effect side_effect.counter = 0 self.sleep_mock.side_effect = side_effect self.assertFalse(self.job_handler._is_job_done()) self.assertRaises(Exception, self.job_handler.push, self.TIMEOUT, self.MAX_RETRIES) self.amqp_client.publish.assert_called_once() self.assertEqual(side_effect.counter, 2) self.assertTrue(self.job_handler._is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.FAILURE) # end test_job_failed def test_execute_job_throws(self): self.assertFalse(self.job_handler._is_job_done()) self.amqp_client.publish.side_effect = \ cfgm_common.exceptions.TimeOutError(500) self.assertRaises(Exception, self.job_handler.push, self.TIMEOUT, self.MAX_RETRIES) self.amqp_client.publish.assert_called_once() self.sleep_mock.assert_not_called() self.assertTrue(self.job_handler._is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.FAILURE) # end test_execute_job_throws def test_max_retries_done(self): job_statuses = [self.JOB_IN_PROGRESS, self.JOB_IN_PROGRESS, self.JOB_IN_PROGRESS] def side_effect(*_): side_effect.counter += 1 _, kwargs = self.amqp_client.add_consumer.call_args_list[0] callback = kwargs['callback'] callback(job_statuses[side_effect.counter-1], self.message) # end side_effect side_effect.counter = 0 self.sleep_mock.side_effect = side_effect self.assertFalse(self.job_handler._is_job_done()) self.assertRaises(Exception, self.job_handler.push, self.TIMEOUT, 3) self.amqp_client.publish.assert_called_once() self.assertEqual(side_effect.counter, 3) self.assertTrue(self.job_handler._is_job_done()) self.assertEqual(self.job_handler.get_job_status(), JobStatus.FAILURE)