def test_execute_job_no_device_json(self): # create job template job_template = JobTemplate( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=True, job_template_playbooks=TestJobManagerUtils.playbooks_list, name='multi_device_no_json') job_template_uuid = self._vnc_lib.job_template_create(job_template) device_id = "aad74e24-a00b-4eb3-8412-f8b9412925c3" TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) job_input_json.update({ "params": { "device_list": [device_id] }, }) jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) + device_id + ",\n" + MsgBundle.getMessage(MsgBundle.PLAYBOOK_RESULTS_MESSAGE) + device_id + ":" + MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) + " \n")
def test_execute_job_run_pb_gen_exc(self): # create job template job_template = JobTemplate( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils.playbooks_list, name='run_pb_gen_exc') job_template_uuid = self._vnc_lib.job_template_create(job_template) TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) exc = Exception('some gen exc') # mock the call to raise exception flexmock(json).should_receive('dumps')\ .and_raise(exc) sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.RUN_PLAYBOOK_ERROR, playbook_uri=TestJobManagerUtils.play_info.playbook_uri, exc_msg=repr(exc)))
def test_execute_job_no_pb_file_on_path(self): # create job template play_info = PlaybookInfoType(playbook_uri='job_manager_test.yaml') playbooks_list = PlaybookInfoListType(playbook_info=[play_info]) job_template = JobTemplate( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils.playbooks_list, name='single_device_template') job_template_uuid = self._vnc_lib.job_template_create(job_template) playbook_info = {"uri": "job_manager_test.yaml"} TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) # mock the call to invoke the playbook process flexmock(os.path).should_receive('exists').and_return(False) sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) + MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND, playbook_uri=playbook_info['uri']))
def run_playbook_process(self, playbook_info, percentage_completed): playbook_process = None self.current_percentage = percentage_completed try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" unique_pb_id = str(uuid.uuid4()) playbook_info['extra_vars']['playbook_input']['unique_pb_id']\ = unique_pb_id exec_id =\ playbook_info['extra_vars']['playbook_input'][ 'job_execution_id'] pr_object_log_start_time = time.time() playbook_process = subprocess32.Popen(["python", playbook_exec_path, "-i", json.dumps(playbook_info)], close_fds=True, cwd='/') # this is to yield the context to the playbooks so that # they start running concurrently time.sleep(0.5) marked_output = self.process_file_and_get_marked_output( unique_pb_id, exec_id, playbook_process ) marked_jsons = self._extract_marked_json(marked_output) self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) pr_object_log_end_time = time.time() self.send_pr_object_log( exec_id, pr_object_log_start_time, pr_object_log_end_time, playbook_process.returncode) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage( MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle. RUN_PLAYBOOK_PROCESS_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id)
def create_job_summary_message(self): job_summary_message = MsgBundle.getMessage( MsgBundle.JOB_SUMMARY_MESSAGE_HDR) if self.job_result_status is None: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_RESULT_STATUS_NONE) elif self.job_result_status == JobStatus.FAILURE: if len(self.failed_device_jobs) > 0: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) for failed_device in self.failed_device_jobs: msg = failed_device + ',' job_summary_message += msg else: job_summary_message += "Job failed. " job_summary_message += "\n" elif self.job_result_status == JobStatus.SUCCESS: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_EXECUTION_COMPLETE) if len(self.job_result) > 0: job_summary_message += MsgBundle.getMessage( MsgBundle.PLAYBOOK_RESULTS_MESSAGE) result_summary = "" for entry in self.job_result: result_summary += \ "%s:%s \n" % (entry, self.job_result[entry]) job_summary_message += result_summary if self.job_result_message is not None: job_summary_message += self.job_result_message return job_summary_message
def run_playbook_process(self, playbook_info, percentage_completed): playbook_process = None self.current_percentage = percentage_completed try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" unique_pb_id = str(uuid.uuid4()) playbook_info['extra_vars']['playbook_input']['unique_pb_id']\ = unique_pb_id exec_id =\ playbook_info['extra_vars']['playbook_input'][ 'job_execution_id'] pr_uve_name = self.get_pr_uve_name_from_device_name(playbook_info) playbook_process = subprocess32.Popen(["python", playbook_exec_path, "-i", json.dumps(playbook_info)], close_fds=True, cwd='/') # Save process ID in case of abort self._pb_pids.append(playbook_process.pid) # this is to yield the context to the playbooks so that # they start running concurrently gevent.sleep(0) marked_output = self.process_file_and_get_marked_output( unique_pb_id, exec_id, playbook_process, pr_uve_name) marked_jsons = self._extract_marked_json(marked_output) playbook_output = marked_jsons.get(JobFileWrite.PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) # create prouter UVE in job_manager only if it is not a multi # device job template if not self.is_multi_device_playbook: status = "SUCCESS" if playbook_process.returncode == 0 \ else "FAILURE" self.send_prouter_uve(exec_id, status) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage( MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) if playbook_output: msg = "%s\n Error Message from playbook: %s" % ( msg, playbook_output.get('message', "") ) raise JobException(msg, self._execution_id) return playbook_output
def test_execute_job_run_pb_process_rc1(self): # create job template job_template = JobTemplate(job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils. playbooks_list, name='run_pb_prc_rc_1') job_template_uuid = self._vnc_lib.job_template_create(job_template) TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) wm = WFManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) loop_var = 0 def mock_readline(): global loop_var loop_var += 1 if loop_var == 1: with open("tests/test.txt", 'r') as f: line = f.readline() return line if loop_var == 2: fake_process.should_receive("poll").and_return(123) loop_var = 0 return "" stdout = flexmock(readline=mock_readline) flexmock(json).should_receive("loads") flexmock(os.path).should_receive('exists').and_return(True) # mock the call to raise exception fake_resp = flexmock(status_code=123) fake_request = flexmock(requests).should_receive( 'post').and_return(fake_resp) fake_process = flexmock(returncode=1, stdout=stdout) fake_process.should_receive('wait') fake_process.should_receive('poll').and_return(123) # flexmock(subprocess).should_receive('TimeoutExpired') flexmock(subprocess32).should_receive('Popen').and_return( fake_process) sys_exit_msg = self.assertRaises(SystemExit, wm.start_job) self.assertEqual(sys_exit_msg.code, MsgBundle.getMessage( MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=TestJobManagerUtils. play_info.playbook_uri) )
def run_playbook_process(self, playbook_info, percentage_completed): playbook_process = None self.current_percentage = percentage_completed try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" unique_pb_id = str(uuid.uuid4()) playbook_info['extra_vars']['playbook_input']['unique_pb_id']\ = unique_pb_id exec_id =\ playbook_info['extra_vars']['playbook_input'][ 'job_execution_id'] pr_uve_name = self.get_pr_uve_name_from_device_name(playbook_info) playbook_process = subprocess32.Popen(["python", playbook_exec_path, "-i", json.dumps(playbook_info)], close_fds=True, cwd='/') # this is to yield the context to the playbooks so that # they start running concurrently gevent.sleep(0) marked_output = self.process_file_and_get_marked_output( unique_pb_id, exec_id, playbook_process, pr_uve_name) marked_jsons = self._extract_marked_json(marked_output) playbook_output = marked_jsons.get(JobFileWrite.PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) # create prouter UVE in job_manager only if it is not a multi # device job template if not self._job_template.get_job_template_multi_device_job(): status = "SUCCESS" if playbook_process.returncode == 0 \ else "FAILURE" self.send_prouter_uve(exec_id, status) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage( MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) if playbook_output: msg = "%s\n Error Message from playbook: %s" % ( msg, playbook_output.get('message', "") ) raise JobException(msg, self._execution_id) return playbook_output
def handle_init_failure(job_input_json, error_num, error_msg): logger.error(MsgBundle.getMessage(error_num, exc_msg=traceback.format_exc())) msg = MsgBundle.getMessage(error_num, exc_msg=error_msg) job_log_utils.send_job_log(job_input_json.get('job_template_fq_name'), job_input_json.get('job_execution_id'), job_input_json.get('fabric_fq_name'), msg, JobStatus.FAILURE) sys.exit(msg)
def create_job_summary_message(self): job_summary_message = MsgBundle.getMessage( MsgBundle.JOB_SUMMARY_MESSAGE_HDR) failed_device_jobs_len = len(self.failed_device_jobs) if self.job_result_status is None: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_RESULT_STATUS_NONE) elif self.job_result_status == JobStatus.FAILURE: if failed_device_jobs_len > 0: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) for failed_device in self.failed_device_jobs: msg = failed_device + ',' job_summary_message += msg else: job_summary_message += "Job failed. " job_summary_message += "\n" elif self.job_result_status == JobStatus.SUCCESS: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_EXECUTION_COMPLETE) device_job_result_len = len(self.job_result) if device_job_result_len > 0: job_summary_message += MsgBundle.getMessage( MsgBundle.PLAYBOOK_RESULTS_MESSAGE) job_summary_message += "Successfully completed "\ "job for %s devices.\n"\ % (device_job_result_len - failed_device_jobs_len) # result_summary would infact be the failed_devices # result summary result_summary = "" device_op_results = [] failed_device_names = [] for entry in self.job_result: if entry in self.failed_device_jobs: result_summary += \ "%s:%s \n" % (self.job_result[entry]['device_name'], self.job_result[entry]['message']) failed_device_names.append( self.job_result[entry]['device_name']) elif self.job_result[entry]['device_op_result']: # could be other device jobs such as device import, topology device_op_results.append( self.job_result[entry]['device_op_result']) if result_summary != "": failed_device_msg = "Job execution failed for %s devices.\n"\ % len(self.failed_device_jobs) result_summary = failed_device_msg + result_summary job_summary_message += result_summary if self.job_result_message is not None: job_summary_message += self.job_result_message return job_summary_message, device_op_results, failed_device_names
def create_job_summary_message(self): job_summary_message = MsgBundle.getMessage( MsgBundle.JOB_SUMMARY_MESSAGE_HDR) failed_device_jobs_len = len(self.failed_device_jobs) if self.job_result_status is None: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_RESULT_STATUS_NONE) elif self.job_result_status == JobStatus.FAILURE: if failed_device_jobs_len > 0: job_summary_message += MsgBundle.getMessage( MsgBundle. JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) for failed_device in self.failed_device_jobs: msg = failed_device + ',' job_summary_message += msg else: job_summary_message += "Job failed. " job_summary_message += "\n" elif self.job_result_status == JobStatus.SUCCESS: job_summary_message += MsgBundle.getMessage( MsgBundle.JOB_EXECUTION_COMPLETE) device_job_result_len = len(self.job_result) if device_job_result_len > 0: job_summary_message += MsgBundle.getMessage( MsgBundle.PLAYBOOK_RESULTS_MESSAGE) job_summary_message += "Successfully completed "\ "job for %s devices.\n"\ % (device_job_result_len - failed_device_jobs_len) # result_summary would infact be the failed_devices # result summary result_summary = "" device_op_results = [] failed_device_names = [] for entry in self.job_result: if entry in self.failed_device_jobs: result_summary += \ "%s:%s \n" % (self.job_result[entry]['device_name'], self.job_result[entry]['message']) failed_device_names.append( self.job_result[entry]['device_name']) elif self.job_result[entry]['device_op_result']: # could be other device jobs such as device import, topology device_op_results.append( self.job_result[entry]['device_op_result']) if result_summary != "": failed_device_msg = "Job execution failed for %s devices.\n"\ % len(self.failed_device_jobs) result_summary = failed_device_msg + result_summary job_summary_message += result_summary if self.job_result_message is not None: job_summary_message += self.job_result_message return job_summary_message, device_op_results, failed_device_names
def test_execute_job_input_schema_ip_not_found(self): # create job template fake_job_template = flexmock( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils. playbooks_list, name='input_schema_template_ip', fq_name=["default-global-system-config", "input_schema_template_ip"], uuid='random_uuid') mock_vnc = flexmock() flexmock(VncApi).new_instances(mock_vnc) flexmock(mock_vnc).should_receive('job_template_read') \ .with_args(id="random_uuid") \ .and_return(fake_job_template) TestJobManagerUtils.mock_sandesh_check() job_input_json = { "job_template_id": "random_uuid", "job_execution_id": TestJobManagerUtils.execution_id, "fabric_fq_name": "Global-system-config:fabric:1", "auth_token": "6e7d7f87faa54fac96a2a28ec752336a", "args": TestJobManagerUtils.args } log_utils = JobLogUtils( sandesh_instance_id=TestJobManagerUtils.execution_id, config_args=json.dumps(TestJobManagerUtils.args)) wm = WFManager(log_utils.get_config_logger(), mock_vnc, job_input_json, log_utils) fake_schema = TestJobManagerUtils.fake_schema fake_job_template.should_receive('get_job_template_input_schema') \ .and_return(fake_schema) fake_job_template.should_receive( 'get_job_template_multi_device_job') \ .and_return(False) fake_job_template.should_receive('get_uuid').and_return('random_uuid') # mock the job_handler to raise an exception fake_job_template.should_receive('get_job_template_playbooks') sys_exit_msg = self.assertRaises(SystemExit, wm.start_job) self.assertEqual(sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_EXC_REC_HDR) + MsgBundle.getMessage( MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND) + " ")
def test_execute_job_handler_gen_exc(self): # create job template fake_job_template = flexmock( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils.playbooks_list, name='single_device_temp_jh_gen_exc', fq_name=[ "default-global-system-config", "single_device_temp_jh_gen_exc" ], uuid='random_uuid') mock_vnc = flexmock() flexmock(VncApi).new_instances(mock_vnc) flexmock(mock_vnc).should_receive('job_template_read')\ .with_args(id="random_uuid")\ .and_return(fake_job_template) TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( "random_uuid") jmgr = JobManager(log_utils.get_config_logger(), mock_vnc, job_input_json, log_utils) fake_job_template.should_receive( 'get_job_template_input_schema').and_return(None) fake_job_template.should_receive( 'get_job_template_multi_device_job').and_return(False) fake_job_template.should_receive('get_uuid').and_return('random_uuid') # mock the job_handler to raise a general exception fake_job_template.should_receive('get_job_template_playbooks')\ .and_raise( Exception("Mock " "Generic Exception in job_handler")) exc = Exception('Mock Generic Exception in job_handler') sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) + MsgBundle.getMessage(MsgBundle.GET_PLAYBOOK_INFO_ERROR, job_template_id="random_uuid", exc_msg=repr(exc)))
def parse_job_input(self, job_input_json): # job input should have job_template_id and execution_id field if job_input_json.get('job_template_id') is None: msg = MsgBundle.getMessage(MsgBundle.JOB_TEMPLATE_MISSING) raise Exception(msg) if job_input_json.get('job_execution_id') is None: msg = MsgBundle.getMessage(MsgBundle.JOB_EXECUTION_ID_MISSING) raise Exception(msg) self.job_template_id = job_input_json.get('job_template_id') self.job_execution_id = job_input_json.get('job_execution_id') self.job_data = job_input_json.get('input') self.fabric_fq_name = job_input_json.get('fabric_fq_name')
def run_playbook_process(self, playbook_info): playbook_process = None try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" playbook_process = subprocess32.Popen([ "python", playbook_exec_path, "-i", json.dumps(playbook_info) ], close_fds=True, cwd='/', stdout=subprocess32.PIPE) marked_output = {} markers = [DEVICE_DATA, PLAYBOOK_OUTPUT] while True: output = playbook_process.stdout.readline() if output == '' and playbook_process.poll() is not None: break if output: self._logger.debug(output) # read device list data and store in variable result handler for marker in markers: if marker in output: marked_output[marker] = output marked_jsons = self._extract_marked_json(marked_output) self._device_data = marked_jsons.get(DEVICE_DATA) self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed): try: # create job log to capture the start of the playbook device_id = \ playbook_info['extra_vars']['playbook_input'].get( 'device_id', "") msg = MsgBundle.getMessage(MsgBundle.START_EXECUTE_PLAYBOOK_MSG, playbook_uri=playbook_info['uri'], device_id=device_id, input_params=json.dumps( playbook_info['extra_vars'] ['playbook_input'] ['input']), extra_params=json.dumps( playbook_info['extra_vars'] ['playbook_input'] ['params'])) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value) if not os.path.exists(playbook_info['uri']): msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id) # Run playbook in a separate process. This is needed since # ansible cannot be used in a greenlet based patched environment self.run_playbook_process(playbook_info, percentage_completed) # create job log to capture completion of the playbook execution msg = MsgBundle.getMessage(MsgBundle.PB_EXEC_COMPLETE_WITH_INFO, playbook_uri=playbook_info['uri'], device_id=device_id) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value) except JobException: raise except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed): playbook_output = None try: # create job log to capture the start of the playbook device_name = \ playbook_info['extra_vars']['playbook_input'].get( 'device_fqname') if device_name: device_name = device_name[-1] playbook_name = playbook_info['uri'].split('/')[-1] msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value, device_name=device_name) if not os.path.exists(playbook_info['uri']): msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id) # Run playbook in a separate process. This is needed since # ansible cannot be used in a greenlet based patched environment playbook_output = self.run_playbook_process( playbook_info, percentage_completed) # create job log to capture completion of the playbook execution msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value, device_name=device_name) return playbook_output except JobException: raise except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id)
def parse_job_input(self, job_input_json): # job input should have job_template_id and execution_id field if job_input_json.get('job_template_id') is None: msg = MsgBundle.getMessage(MsgBundle.JOB_TEMPLATE_MISSING) raise Exception(msg) if job_input_json.get('job_execution_id') is None: msg = MsgBundle.getMessage( MsgBundle.JOB_EXECUTION_ID_MISSING) raise Exception(msg) self.job_template_id = job_input_json.get('job_template_id') self.job_execution_id = job_input_json.get('job_execution_id') self.job_data = job_input_json.get('input') self.fabric_fq_name = job_input_json.get('fabric_fq_name')
def start_job(self): # spawn job greenlets job_handler = JobHandler(self._logger, self._vnc_api, self.job_template, self.job_execution_id, self.job_data, self.job_utils, self.device_json, self.auth_token, self.contrail_cluster_id, self.api_server_host, self.job_log_utils, self.sandesh_args, self.fabric_fq_name, self.job_log_utils.args.playbook_timeout, self.playbook_seq, self.vnc_api_init_params, self._zk_client) # check if its a multi device playbook playbooks = self.job_template.get_job_template_playbooks() play_info = playbooks.playbook_info[self.playbook_seq] is_multi_device_playbook = play_info.multi_device_playbook # for fabric config push as part of delete workflow, # device json is not needed. There will be no performance # impact as fabric delete from DM will always have one prouter # uuid in the device_list. if is_multi_device_playbook: if self.device_json is None or not self.device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self.job_execution_id) else: self.handle_multi_device_job(job_handler, self.result_handler) else: self.handle_single_job(job_handler, self.result_handler)
def _validate_job_input(self, input_schema, ip_json): if ip_json is None: msg = MsgBundle.getMessage(MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND) raise JobException(msg, self.job_execution_id) try: ip_schema_json = input_schema if isinstance(input_schema, str): ip_schema_json = json.loads(input_schema) jsonschema.validate(ip_json, ip_schema_json) self._logger.debug("Input Schema Validation Successful" "for template %s" % self.job_template_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA, job_template_id=self.job_template_id, exc_obj=exp) raise JobException(msg, self.job_execution_id)
def handle_job(self, result_handler, job_percent_per_task, device_id=None): try: msg = "Starting playbook execution for job template %s with " \ "execution id %s" % (self._job_template.get_uuid(), self._execution_id) self._logger.debug(msg) # get the playbook information from the job template playbook_info = self.get_playbook_info(job_percent_per_task, device_id) # run the playbook self.run_playbook(playbook_info) msg = MsgBundle.getMessage( MsgBundle.PLAYBOOK_EXECUTION_COMPLETE, job_template_id=self._job_template.get_uuid(), job_execution_id=self._execution_id) self._logger.debug(msg) result_handler.update_job_status(JobStatus.SUCCESS, msg, device_id) self.update_result_handler(result_handler) except JobException as job_exp: self._logger.error("%s" % job_exp.msg) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg, device_id) except Exception as exp: self._logger.error("Error while executing job %s " % repr(exp)) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, exp.message, device_id)
def start_job(self): # spawn job greenlets job_handler = JobHandler(self._logger, self._vnc_api, self.job_template, self.job_execution_id, self.job_data, self.job_utils, self.device_json, self.auth_token, self.contrail_cluster_id, self.api_server_host, self.job_log_utils, self.sandesh_args, self.fabric_fq_name, self.job_log_utils.args.playbook_timeout, self.playbook_seq, self.vnc_api_init_params, self._zk_client) self.job_handler = job_handler # check if its a multi device playbook playbooks = self.job_template.get_job_template_playbooks() play_info = playbooks.playbook_info[self.playbook_seq] is_multi_device_playbook = play_info.multi_device_playbook # for fabric config push as part of delete workflow, # device json is not needed. There will be no performance # impact as fabric delete from DM will always have one prouter # uuid in the device_list. if is_multi_device_playbook: if self.device_json is None or not self.device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self.job_execution_id) else: self.handle_multi_device_job(job_handler, self.result_handler) else: self.handle_single_job(job_handler, self.result_handler)
def send_job_log(self, job_template_fqname, job_execution_id, fabric_fq_name, message, status, completion_percent=None, result=None, timestamp=None, device_name=None, details=None): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) if timestamp is None: timestamp = int(round(time.time() * 1000)) details_str = json.dumps(details) if details else None job_log_entry = JobLogEntry( name=job_template_fqname, execution_id=job_execution_id, fabric_name=fabric_fq_name, timestamp=timestamp, message=message, status=status, percentage_completed=completion_percent, result=result, device_name=device_name, details=details_str) job_log = JobLog(log_entry=job_log_entry) job_log.send(sandesh=self.config_logger._sandesh) self.config_logger.debug("Created job log for job template: %s, " " execution id: %s, fabric_fq_name: %s" "status: %s, completion_percent %s, " "result: " "%s, message: %s" % (job_template_fqname, job_execution_id, fabric_fq_name, status, completion_percent, result, message)) except Exception as e: msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_LOG_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, fabric_name=fabric_fq_name, exc_msg=repr(e)) raise JobException(msg, job_execution_id)
def send_job_execution_uve( self, fabric_fq_name, job_template_fqname, job_execution_id, timestamp=None, percentage_completed=None): try: fabric_job_name = list(job_template_fqname) fabric_job_name.insert(0, fabric_fq_name) fabric_job_uve_name = ':'.join(map(str, fabric_job_name)) job_exe_data = FabricJobExecution( name=fabric_job_uve_name, job_status='IN_PROGRESS', percentage_completed=percentage_completed) job_uve = FabricJobUve( data=job_exe_data, sandesh=self.config_logger._sandesh) job_uve.send(sandesh=self.config_logger._sandesh) except Exception as exp: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def send_prouter_job_uve(self, job_template_fqname, fq_names, job_execution_id, prouter_state=None, job_status=None): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) if prouter_state is None: prouter_job_data = PhysicalRouterJobExecution( name=fq_names, execution_id=job_execution_id, job_status=job_status) prouter_job_data = PhysicalRouterJobExecution( name=fq_names, execution_id=job_execution_id, prouter_state=prouter_state, job_status=job_status) prouter_job_uve = PhysicalRouterJobUve( data=prouter_job_data, sandesh=self.config_logger._sandesh) prouter_job_uve.send(sandesh=self.config_logger._sandesh) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def test_execute_job_generic_exception(self): # create job template job_template = JobTemplate(job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils. playbooks_list, name='single_device_template_gen_exc') job_template_uuid = self._vnc_lib.job_template_create(job_template) TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) wm = WFManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) # mock the job_handler to raise a general exception mock_job_handler = flexmock() flexmock(JobHandler).new_instances(mock_job_handler) exc = Exception("Mock " "Generic Exception") flexmock(mock_job_handler).should_receive('handle_job') \ .and_raise(exc) exc_msg = repr(exc) sys_exit_msg = self.assertRaises(SystemExit, wm.start_job) self.assertEqual(sys_exit_msg.code, MsgBundle.getMessage( MsgBundle.EXC_JOB_ERR_HDR) + exc_msg + " ")
def test_execute_job_no_template_id(self): job_input_json = {"no_template_id": "Missing template_id"} exc_msg = self.assertRaises(Exception, JobManager, None, None, job_input_json, None) self.assertEqual(str(exc_msg), MsgBundle.getMessage(MsgBundle.JOB_TEMPLATE_MISSING))
def initialize_sandesh_logger(self, config_args, sandesh=True, sandesh_instance=None): # parse the logger args args = self.parse_logger_args(config_args) args.random_collectors = args.collectors if args.collectors: args.random_collectors = random.sample(args.collectors, len(args.collectors)) self.args = args # initialize logger logger = JobLogger(args=args, sandesh_instance_id=self.sandesh_instance_id, sandesh_instance=sandesh_instance) if not sandesh_instance and sandesh: try: sandesh_util = SandeshUtils(logger) sandesh_util.wait_for_connection_establish() except JobException: msg = MsgBundle.getMessage( MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR) raise JobException(msg) logger.info("Sandesh is initialized." " Config logger instance created.") return logger
def send_job_execution_uve(self, fabric_fq_name, job_template_fqname, job_execution_id, timestamp=None, percentage_completed=None): try: fabric_job_name = list(job_template_fqname) fabric_job_name.insert(0, fabric_fq_name) fabric_job_uve_name = ':'.join(map(str, fabric_job_name)) job_exe_data = FabricJobExecution( name=fabric_job_uve_name, job_status='IN_PROGRESS', percentage_completed=percentage_completed) job_uve = FabricJobUve(data=job_exe_data, sandesh=self.config_logger._sandesh) job_uve.send(sandesh=self.config_logger._sandesh) except Exception as exp: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def handle_job(self, result_handler, job_percent_per_task, device_id=None, device_name=None): playbook_output = None playbook_info = None try: msg = "Starting playbook execution for job template %s with " \ "execution id %s" % (self._job_template.get_uuid(), self._execution_id) self._logger.debug(msg) # get the playbook information from the job template playbook_info = self.get_playbook_info(job_percent_per_task, device_id) # run the playbook and retrieve the playbook output if any playbook_output = self.run_playbook( playbook_info, result_handler.percentage_completed) # retrieve the device_op_results in case it was set for # generic device operations. playbook_output_results = None if playbook_output != None: playbook_output_results = playbook_output.get('results') msg = MsgBundle.getMessage( MsgBundle.PLAYBOOK_EXECUTION_COMPLETE, job_template_name=self._job_template.get_fq_name()[-1], job_execution_id=self._execution_id) self._logger.debug(msg) result_handler.update_job_status(JobStatus.SUCCESS, msg, device_id, device_name, pb_results=playbook_output_results) if playbook_output: result_handler.update_playbook_output(playbook_output) self.check_and_send_prouter_job_uve_for_multidevice(playbook_info, JobStatus.SUCCESS.value, playbook_output_results) if self.current_percentage: result_handler.percentage_completed = self.current_percentage except JobException as job_exp: self._logger.error("%s" % job_exp.msg) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg, device_id, device_name) if playbook_info: self.check_and_send_prouter_job_uve_for_multidevice(playbook_info, JobStatus.FAILURE.value) except Exception as exp: self._logger.error("Error while executing job %s " % repr(exp)) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, exp.message, device_id, device_name) if playbook_info: self.check_and_send_prouter_job_uve_for_multidevice(playbook_info, JobStatus.FAILURE.value)
def test_execute_job_no_execution_id(self): job_input_json = {"job_template_id": "random_template_id"} exc_msg = self.assertRaises(Exception, JobManager, None, None, job_input_json, None) self.assertEqual( str(exc_msg), MsgBundle.getMessage(MsgBundle.JOB_EXECUTION_ID_MISSING))
def test_execute_job_input_schema(self): # create job template fake_job_template = flexmock( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils. playbooks_list, name='input_schema_template', fq_name=["default-global-system-config", "input_schema_template"], uuid='random_uuid') TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( "random_uuid") mock_vnc = flexmock() flexmock(VncApi).new_instances(mock_vnc) flexmock(mock_vnc).should_receive( 'job_template_read').with_args( id="random_uuid").and_return(fake_job_template) wm = WFManager(log_utils.get_config_logger(), mock_vnc, job_input_json, log_utils) exc = Exception("'name' is a required property") fake_schema = TestJobManagerUtils.fake_schema fake_job_template.should_receive( 'get_job_template_input_schema').and_return(fake_schema) fake_job_template.should_receive('get_uuid').and_return( 'random_uuid') # mock the job_handler to raise an exception fake_job_template.should_receive('get_job_template_playbooks') sys_exit_msg = self.assertRaises(SystemExit, wm.start_job) self.assertEqual(sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_EXC_REC_HDR) + MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA, job_template_id="random_uuid", exc_obj=exc) + " ")
def test_execute_job_no_credentials(self): # create job template job_template = JobTemplate(job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=True, job_template_playbooks=TestJobManagerUtils. playbooks_list, name='multi_device_no_credentials') job_template_uuid = self._vnc_lib.job_template_create(job_template) device_id = "aad74e24-a00b-4eb3-8412-f8b9412925c3" TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) job_input_json.update({"params": {"device_list": [device_id]}, "device_json": { device_id: {"device_family": "MX", "device_vendor": "Juniper", "device_product": "Some random product", "device_fqname": [ "default-global-system-config", "random_fqname"]}}}) wm = WFManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) sys_exit_msg = self.assertRaises(SystemExit, wm.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage(MsgBundle. JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) + device_id + ",\n" + MsgBundle.getMessage(MsgBundle.PLAYBOOK_RESULTS_MESSAGE) + device_id + ":" + MsgBundle.getMessage( MsgBundle.NO_CREDENTIALS_FOUND, device_id=device_id) + " \n")
def _validate_job_input(self, input_schema, ip_json): if ip_json is None: msg = MsgBundle.getMessage( MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND) raise JobException(msg, self.job_execution_id) try: ip_schema_json = input_schema if isinstance(input_schema, str): ip_schema_json = json.loads(input_schema) jsonschema.validate(ip_json, ip_schema_json) self._logger.debug("Input Schema Validation Successful" "for template %s" % self.job_template_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA, job_template_id=self.job_template_id, exc_obj=exp) raise JobException(msg, self.job_execution_id)
def close_sandesh_connection(self): try: self.wait_for_msg_send() except JobException as job_exp: msg = MsgBundle.getMessage(MsgBundle.CLOSE_SANDESH_EXCEPTION) self._logger.error(msg) job_exp.msg = msg raise job_exp finally: self.uninit_sandesh()
def test_execute_job_explicit_mismatch(self): # create job template job_template = JobTemplate( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=True, job_template_playbooks=TestJobManagerUtils.playbooks_list, name='multi_device_explicit_mismatch') job_template_uuid = self._vnc_lib.job_template_create(job_template) TestJobManagerUtils.mock_sandesh_check() device_id = "aad74e24-a00b-4eb3-8412-f8b9412925c3" job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) job_input_json.update({ "params": { "device_list": [device_id] }, "device_json": { device_id: { "device_family": "MX", "device_vendor": "Arista", "device_username": "******", "device_password": "******" } } }) jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage( MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) + device_id + ",\n" + MsgBundle.getMessage(MsgBundle.PLAYBOOK_RESULTS_MESSAGE) + device_id + ":" + MsgBundle.getMessage(MsgBundle.PLAYBOOK_INFO_DEVICE_MISMATCH, device_vendor="Arista", device_family="MX") + " \n")
def run_playbook(self, playbook_info, percentage_completed): playbook_output = None try: # create job log to capture the start of the playbook device_name = \ playbook_info['extra_vars']['playbook_input'].get( 'device_fqname') if device_name: device_name = device_name[-1] playbook_name = playbook_info['uri'].split('/')[-1] msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) if not os.path.exists(playbook_info['uri']): msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id) # Run playbook in a separate process. This is needed since # ansible cannot be used in a greenlet based patched environment playbook_output = self.run_playbook_process(playbook_info, percentage_completed) # create job log to capture completion of the playbook execution msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) return playbook_output except JobException: raise except Exception as exp: trace = traceback.format_exc() msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException("%s\n%s" %(msg, trace), self._execution_id)
def send_prouter_object_log(self, prouter_fqname, job_execution_id, job_input, job_template_fqname, onboarding_state, os_version=None, serial_num=None, timestamp=None): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) prouter_fqname = self.get_fq_name_log_str(prouter_fqname) if timestamp is None: timestamp = int(round(time.time() * 1000)) # create the prouter object log prouter_log_entry = PRouterOnboardingLogEntry( name=prouter_fqname, job_execution_id=job_execution_id, os_version=os_version, serial_num=serial_num, onboarding_state=onboarding_state, timestamp=timestamp, job_template_fqname=job_template_fqname, job_input=job_input) prouter_log = PRouterOnboardingLog(log_entry=prouter_log_entry) prouter_log.send(sandesh=self.config_logger._sandesh) self.config_logger.debug( "Created prouter object log for router: %s, " " execution id: %s, job_template: %s, os_version: " "%s, serial_num: %s, onboarding_state %s" % (prouter_fqname, job_execution_id, job_template_fqname, os_version, serial_num, onboarding_state)) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.SEND_PROUTER_OBJECT_LOG_ERROR, prouter_fqname=prouter_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def start_job(self): # spawn job greenlets job_handler = JobHandler(self._logger, self._vnc_api, self.job_template, self.job_execution_id, self.job_data, self.job_utils, self.device_json, self.auth_token, self.api_server_host, self.job_log_utils, self.sandesh_args, self.fabric_fq_name, self.job_log_utils.args.playbook_timeout, self.playbook_seq, self.vnc_api_init_params, self._zk_client, self.db_init_params, self.cluster_id) if self.device_json is not None: if not self.device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self.job_execution_id) else: self.handle_multi_device_job(job_handler, self.result_handler) else: self.handle_single_job(job_handler, self.result_handler)
def send_prouter_job_uve( self, job_template_fqname, fq_names, job_execution_id, prouter_state=None, job_status=None, percentage_completed=None, device_op_results = "{}"): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) if prouter_state is None: prouter_job_data = PhysicalRouterJobExecution( name=fq_names, execution_id=job_execution_id, job_status=job_status, percentage_completed=percentage_completed, device_op_results = device_op_results ) else: prouter_job_data = PhysicalRouterJobExecution( name=fq_names, execution_id=job_execution_id, prouter_state=prouter_state, job_status=job_status, percentage_completed=percentage_completed, device_op_results = device_op_results ) prouter_job_uve = PhysicalRouterJobUve( data=prouter_job_data, sandesh=self.config_logger._sandesh) prouter_job_uve.send(sandesh=self.config_logger._sandesh) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def initialize_sandesh_logger(self, config_args, sandesh=True, sandesh_instance=None): # parse the logger args args = self.parse_logger_args(config_args) args.random_collectors = args.collectors if args.collectors: args.random_collectors = random.sample(args.collectors, len(args.collectors)) self.args = args # initialize logger logger = JobLogger(args=args, sandesh_instance_id=self.sandesh_instance_id, sandesh_instance=sandesh_instance) if not sandesh_instance and sandesh: try: sandesh_util = SandeshUtils(logger) sandesh_util.wait_for_connection_establish() except JobException: msg = MsgBundle.getMessage( MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR) raise JobException(msg) logger.info("Sandesh is initialized. Config logger instance created.") return logger
def handle_job(self, result_handler, job_percent_per_task, device_id=None, device_name=None): playbook_output = None playbook_info = None try: msg = "Starting playbook execution for job template %s with " \ "execution id %s" % (self._job_template.get_uuid(), self._execution_id) self._logger.debug(msg) # Always acquire the lock while executing the multi device jobs if device_id is not None: if not self._acquire_device_lock(device_id): raise JobException(MsgBundle.getMessage( MsgBundle.DEVICE_LOCK_FAILURE)) # get the playbook information from the job template playbook_info = self.get_playbook_info(job_percent_per_task, device_id) # run the playbook and retrieve the playbook output if any playbook_output = self.run_playbook( playbook_info, result_handler.percentage_completed) # retrieve the device_op_results in case it was set for # generic device operations. playbook_output_results = None if playbook_output != None: playbook_output_results = playbook_output.get('results') msg = MsgBundle.getMessage( MsgBundle.PLAYBOOK_EXECUTION_COMPLETE, job_template_name=self._job_template.get_fq_name()[-1], job_execution_id=self._execution_id) self._logger.debug(msg) result_handler.update_job_status( JobStatus.SUCCESS, msg, device_id, device_name, pb_results=playbook_output_results) if playbook_output: result_handler.update_playbook_output(playbook_output) self.check_and_send_prouter_job_uve_for_multidevice( playbook_info, JobStatus.SUCCESS.value, playbook_output_results) if self.current_percentage: result_handler.percentage_completed = self.current_percentage except JobException as job_exp: self._logger.error("%s" % job_exp.msg) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg, device_id, device_name) if playbook_info: self.check_and_send_prouter_job_uve_for_multidevice( playbook_info, JobStatus.FAILURE.value) except Exception as exp: self._logger.error("Error while executing job %s " % repr(exp)) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, exp.message, device_id, device_name) if playbook_info: self.check_and_send_prouter_job_uve_for_multidevice( playbook_info, JobStatus.FAILURE.value) finally: if device_id is not None: self._release_device_lock(device_id)
def get_playbook_info(self, job_percent_per_task, device_id=None): try: # create the cmd line param for the playbook extra_vars = { 'input': self._job_input, 'job_template_id': self._job_template.get_uuid(), 'job_template_fqname': self._job_template.fq_name, 'fabric_fq_name': self._fabric_fq_name, 'auth_token': self._auth_token, 'api_server_host': self._api_server_host, 'job_execution_id': self._execution_id, 'args': self._sandesh_args, 'vnc_api_init_params': self._vnc_api_init_params, 'db_init_params': self._db_init_params, 'cluster_id': self._cluster_id, 'playbook_job_percentage': job_percent_per_task } playbooks = self._job_template.get_job_template_playbooks() if device_id: if not self._device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self._execution_id) device_data = self._device_json.get(device_id) if not device_data: msg = MsgBundle.getMessage(MsgBundle.NO_DEVICE_DATA_FOUND, device_id=device_id) raise JobException(msg, self._execution_id) device_family = device_data.get('device_family') device_vendor = device_data.get('device_vendor') device_product = device_data.get('device_product') if not device_vendor or not device_family: msg = MsgBundle.getMessage(MsgBundle. DEVICE_VENDOR_FAMILY_MISSING, device_id=device_id) raise JobException(msg, self._execution_id) if not device_product: msg = MsgBundle.getMessage(MsgBundle. PRODUCT_NAME_MISSING, device_id=device_id) raise JobException(msg, self._execution_id) # check for credentials,required param; else playbooks # will fail device_username = device_data.get('device_username') device_password = device_data.get('device_password') if not device_username or not device_password: msg = MsgBundle.getMessage(MsgBundle. NO_CREDENTIALS_FOUND, device_id=device_id) raise JobException(msg, self._execution_id) # update extra-vars to reflect device-related params device_fqname = device_data.get('device_fqname') device_management_ip = device_data.get('device_management_ip') image_uuid = device_data.get('device_image_uuid') hitless_upgrade = device_data.get('device_hitless_upgrade') extra_vars.update({ 'device_id': device_id, 'device_fqname': device_fqname, 'device_management_ip': device_management_ip, 'vendor': device_vendor, 'device_family': device_family, 'device_username': device_username, 'device_password': device_password, 'product_name': device_product, 'device_image_uuid': image_uuid, 'device_hitless_upgrade': hitless_upgrade }) self._logger.debug("Passing the following device " "ip to playbook %s " % device_management_ip) # get the playbook uri from the job template play_info = playbooks.playbook_info[self._playbook_seq] playbook_input = {'playbook_input': extra_vars} playbook_info = dict() playbook_info['uri'] = play_info.playbook_uri playbook_info['extra_vars'] = playbook_input return playbook_info except JobException: raise except Exception as exp: msg = MsgBundle.getMessage( MsgBundle.GET_PLAYBOOK_INFO_ERROR, job_template_id=self._job_template.get_uuid(), exc_msg=repr(exp)) raise JobException(msg, self._execution_id)
def parse_args(): parser = argparse.ArgumentParser(description='Ansible playbook input ' 'parameters') parser.add_argument('-i', '--playbook_input', nargs=1, help='Playbook input json') return parser.parse_args() if __name__ == "__main__": playbook_input_json = None try: playbook_params = parse_args() playbook_input_json = json.loads(playbook_params.playbook_input[0]) if playbook_input_json is None: sys.exit(MsgBundle.getMessage(MsgBundle.NO_PLAYBOOK_INPUT_DATA)) except Exception as exp: ERR_MSG = "Failed to start playbook due "\ "to Exception: %s" % traceback.print_stack() JM_LOGGER.error(ERR_MSG) sys.exit(MsgBundle.getMessage(MsgBundle.PLAYBOOK_INPUT_PARSING_ERROR, exc_msg=repr(exp))) playbook_helper = PlaybookHelper() pb_output = playbook_helper.execute_playbook(playbook_input_json) # if it comes here, it implies the pb_output is of correct # format and is present with status Sucess. So derive the # playbook output to be written to file and finally write END to the file unique_pb_id = playbook_input_json['extra_vars'][ 'playbook_input']['unique_pb_id']
def execute_playbook(self, playbook_info): output = None try: loader = DataLoader() inventory = InventoryManager(loader=loader, sources=['localhost']) variable_manager = VariableManager(loader=loader, inventory=inventory) Options = namedtuple('Options', ['listtags', 'listtasks', 'listhosts', 'syntax', 'connection', 'module_path', 'forks', 'remote_user', 'private_key_file', 'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args', 'scp_extra_args', 'become', 'become_method', 'become_user', 'verbosity', 'check', 'diff']) options = Options(listtags=False, listtasks=False, listhosts=False, syntax=False, connection='ssh', module_path=None, forks=100, remote_user=None, private_key_file=None, ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None, scp_extra_args=None, become=None, become_method=None, become_user=None, verbosity=None, check=False, diff=False) variable_manager.extra_vars = playbook_info['extra_vars'] pbex = PlaybookExecutor(playbooks=[playbook_info['uri']], inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=None) ret_val = pbex.run() output = self.get_plugin_output(pbex) if ret_val != 0: msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_RETURN_WITH_ERROR) raise Exception(msg) if output is None or output.get('status') is None: msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_OUTPUT_MISSING) raise Exception(msg) if output.get('status').lower() == "failure": msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_STATUS_FAILED) raise Exception(msg) return output except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXECUTE_ERROR, playbook_uri=playbook_info['uri'], execution_id=playbook_info['extra_vars'] ['playbook_input']['job_execution_id'], exc_msg=repr(exp)) if exp.message: msg = msg + "\n" + exp.message JM_LOGGER.error(msg) # after handling exception, write an END # to stop listening to the file if created unique_pb_id = playbook_info['extra_vars'][ 'playbook_input']['unique_pb_id'] exec_id = playbook_info['extra_vars']['playbook_input'][ 'job_execution_id'] self._job_file_write.write_to_file( exec_id, unique_pb_id, JobFileWrite.PLAYBOOK_OUTPUT, json.dumps(output) ) with open("/tmp/"+exec_id, "a") as f: f.write(unique_pb_id + 'END' + PLAYBOOK_EOL_PATTERN) sys.exit(msg)
def start_job(self): job_error_msg = None job_template = None try: # create job UVE and log self.result_handler = JobResultHandler(self.job_template_id, self.job_execution_id, self.fabric_fq_name, self._logger, self.job_utils, self.job_log_utils) job_template = self.job_utils.read_job_template() msg = MsgBundle.getMessage(MsgBundle.START_JOB_MESSAGE, job_execution_id=self.job_execution_id, job_template_name=\ job_template.fq_name[-1]) self._logger.debug(msg) timestamp = int(round(time.time() * 1000)) self.job_log_utils.send_job_log(job_template.fq_name, self.job_execution_id, self.fabric_fq_name, msg, JobStatus.STARTING.value, timestamp=timestamp) # validate job input if required by job_template input_schema input_schema = job_template.get_job_template_input_schema() if input_schema: self._validate_job_input(input_schema, self.job_data) playbook_list = job_template.get_job_template_playbooks()\ .get_playbook_info() job_percent = None # calculate job percentage for each playbook if len(playbook_list) > 1: task_weightage_array = [ pb_info.job_completion_weightage for pb_info in playbook_list] for i in range(0, len(playbook_list)): if len(playbook_list) > 1: # get the job percentage based on weightage of each plabook # when they are chained job_percent = \ self.job_log_utils.calculate_job_percentage( len(playbook_list), buffer_task_percent=True, total_percent=100, task_seq_number=i + 1, task_weightage_array=task_weightage_array)[0] else: job_percent = \ self.job_log_utils.calculate_job_percentage( len(playbook_list), buffer_task_percent=True, total_percent=100)[0] # using equal weightage retry_devices = None while True: job_mgr = JobManager(self._logger, self._vnc_api, self.job_input, self.job_log_utils, job_template, self.result_handler, self.job_utils, i, job_percent, self._zk_client, self.db_init_params, self.cluster_id) job_mgr.start_job() # retry the playbook execution if retry_devices is added to # the playbook output job_status = self.result_handler.job_result_status retry_devices = self.result_handler.get_retry_devices() if job_status == JobStatus.FAILURE or not retry_devices: break self.job_input['device_json'] = retry_devices # stop the workflow if playbook failed if self.result_handler.job_result_status == JobStatus.FAILURE: # stop workflow only if its a single device job or # it is a multi device playbook # and all the devices have failed some job execution # declare it as failure and the stop the workflow if self.job_input.get('device_json') is None or\ len(self.result_handler.failed_device_jobs)\ == len(self.job_input.get('device_json')): self._logger.error( "Stop the workflow on the failed Playbook.") break elif not retry_devices: # it is a multi device playbook but one of the device jobs # have failed. This means we should still declare # the operation as success. We declare workflow as # success even if one of the devices has succeeded the job self.result_handler.job_result_status = JobStatus.SUCCESS # update the job input with marked playbook output json pb_output = self.result_handler.playbook_output or {} # read the device_data output of the playbook # and update the job input so that it can be used in next # iteration if not self.job_input.get('device_json'): device_json = pb_output.pop('device_json', None) self.job_input['device_json'] = device_json self.job_input.get('input', {}).update(pb_output) # create job completion log and update job UVE self.result_handler.create_job_summary_log( job_template.fq_name) # in case of failures, exit the job manager process with failure if self.result_handler.job_result_status == JobStatus.FAILURE: job_error_msg = self.result_handler.job_summary_message except JobException as exp: err_msg = "Job Exception recieved: %s " % repr(exp) self._logger.error(err_msg) self._logger.error("%s" % traceback.format_exc()) self.result_handler.update_job_status(JobStatus.FAILURE, err_msg) if job_template: self.result_handler.create_job_summary_log( job_template.fq_name) job_error_msg = err_msg except Exception as exp: err_msg = "Error while executing job %s " % repr(exp) self._logger.error(err_msg) self._logger.error("%s" % traceback.format_exc()) self.result_handler.update_job_status(JobStatus.FAILURE, err_msg) self.result_handler.create_job_summary_log(job_template.fq_name) job_error_msg = err_msg finally: # need to wait for the last job log and uve update to complete # via sandesh and then close sandesh connection sandesh_util = SandeshUtils(self._logger) sandesh_util.close_sandesh_connection() self._logger.info("Closed Sandesh connection") if job_error_msg is not None: sys.exit(job_error_msg)