def run_playbook_process(self, playbook_info, percentage_completed): playbook_process = None self.current_percentage = percentage_completed try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" unique_pb_id = str(uuid.uuid4()) playbook_info['extra_vars']['playbook_input']['unique_pb_id']\ = unique_pb_id exec_id =\ playbook_info['extra_vars']['playbook_input'][ 'job_execution_id'] pr_object_log_start_time = time.time() playbook_process = subprocess32.Popen(["python", playbook_exec_path, "-i", json.dumps(playbook_info)], close_fds=True, cwd='/') # this is to yield the context to the playbooks so that # they start running concurrently time.sleep(0.5) marked_output = self.process_file_and_get_marked_output( unique_pb_id, exec_id, playbook_process ) marked_jsons = self._extract_marked_json(marked_output) self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) pr_object_log_end_time = time.time() self.send_pr_object_log( exec_id, pr_object_log_start_time, pr_object_log_end_time, playbook_process.returncode) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage( MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle. RUN_PLAYBOOK_PROCESS_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id)
def read_fabric_data(self, request_params, job_execution_id, is_delete=False): if request_params.get('input') is None: err_msg = "Missing job input" raise JobException(err_msg, job_execution_id) fabric_fq_name = None if request_params.get('input').get('fabric_fq_name'): fabric_fq_name = request_params.get('input').get('fabric_fq_name') elif request_params.get('input').get('fabric_uuid'): # get the fabric fq_name from the db if fabric_uuid is provided fabric_uuid = request_params.get('input').get('fabric_uuid') try: fabric_fq_name = self._db_conn.uuid_to_fq_name(fabric_uuid) except NoIdError as e: raise JobException(str(e), job_execution_id) else: if "device_deletion_template" in request_params.get( 'job_template_fq_name'): fabric_fq_name = ["__DEFAULT__"] elif not is_delete: err_msg = "Missing fabric details in the job input" raise JobException(err_msg, job_execution_id) if fabric_fq_name: fabric_fq_name_str = ':'.join(map(str, fabric_fq_name)) request_params['fabric_fq_name'] = fabric_fq_name_str
def run_playbook_process(self, playbook_info, percentage_completed): playbook_process = None self.current_percentage = percentage_completed try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" unique_pb_id = str(uuid.uuid4()) playbook_info['extra_vars']['playbook_input']['unique_pb_id']\ = unique_pb_id exec_id =\ playbook_info['extra_vars']['playbook_input'][ 'job_execution_id'] pr_uve_name = self.get_pr_uve_name_from_device_name(playbook_info) playbook_process = subprocess32.Popen(["python", playbook_exec_path, "-i", json.dumps(playbook_info)], close_fds=True, cwd='/') # Save process ID in case of abort self._pb_pids.append(playbook_process.pid) # this is to yield the context to the playbooks so that # they start running concurrently gevent.sleep(0) marked_output = self.process_file_and_get_marked_output( unique_pb_id, exec_id, playbook_process, pr_uve_name) marked_jsons = self._extract_marked_json(marked_output) playbook_output = marked_jsons.get(JobFileWrite.PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) # create prouter UVE in job_manager only if it is not a multi # device job template if not self.is_multi_device_playbook: status = "SUCCESS" if playbook_process.returncode == 0 \ else "FAILURE" self.send_prouter_uve(exec_id, status) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage( MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle. PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) if playbook_output: msg = "%s\n Error Message from playbook: %s" % ( msg, playbook_output.get('message', "") ) raise JobException(msg, self._execution_id) return playbook_output
def calculate_job_percentage(self, num_tasks, buffer_task_percent=False, task_seq_number=None, total_percent=100, task_weightage_array=None): if num_tasks is None: raise JobException("Number of tasks is required to calculate " "the job percentage") try: getcontext().prec = 3 # Use buffered approach to mitigate the cumulative task percentages # exceeding the total job percentage if buffer_task_percent: buffer_percent = 0.05 * total_percent total_percent -= buffer_percent # if task weightage is not provided, the task will recive an # equally divided chunk from the total_percent based on the total # number of tasks if task_weightage_array is None: success_task_percent = float(old_div(Decimal(total_percent), Decimal(num_tasks))) else: if task_seq_number is None: raise JobException("Unable to calculate the task " "percentage since the task sequence " "number is not provided") success_task_percent = float(old_div(Decimal( task_weightage_array[task_seq_number - 1] * total_percent), 100)) # based on the task sequence number calculate the percentage to be # marked in cases of error. This is required to mark the job to # 100% completion in cases of errors when successor tasks will not # be executed. failed_task_percent = None if task_seq_number: if task_weightage_array is None: failed_task_percent = (num_tasks - task_seq_number + 1) *\ success_task_percent else: failed_task_percent = 0.00 for task_index in range(task_seq_number, num_tasks): task_percent = float(old_div(Decimal( task_weightage_array[task_index - 1] * total_percent), 100)) failed_task_percent += task_percent self.config_logger.info("success_task_percent %s " "failed_task_percent %s " % (success_task_percent, failed_task_percent)) return success_task_percent, failed_task_percent except Exception as e: msg = "Exception while calculating the job pecentage %s " % repr(e) self.config_logger.error(msg) self.config_logger.error("%s" % traceback.format_exc()) raise JobException(e)
def read_device_data(self, device_list, request_params, job_exec_id): device_data = dict() for device_id in device_list: try: (ok, result) = self.db_read("physical-router", device_id, [ 'physical_router_user_credentials', 'physical_router_management_ip', 'fq_name', 'physical_router_device_family', 'physical_router_vendor_name', 'physical_router_product_name', 'fabric_refs' ]) if not ok: msg = "Error while reading the physical router " \ "with id %s : %s" % (device_id, result) raise JobException(msg, job_exec_id) except Exception as e: msg = "Exception while reading device %s %s " % \ (device_id, str(e)) raise JobException(msg, job_exec_id) device_json = { "device_management_ip": result.get('physical_router_management_ip') } device_json.update({"device_fqname": result.get('fq_name')}) user_cred = result.get('physical_router_user_credentials') if user_cred: device_json.update( {"device_username": user_cred.get('username')}) device_json.update( {"device_password": user_cred.get('password')}) device_family = result.get("physical_router_device_family") if device_family: device_json.update({"device_family": device_family}) device_vendor_name = result.get("physical_router_vendor_name") if device_vendor_name: device_json.update({"device_vendor": device_vendor_name}) device_product_name = result.get("physical_router_product_name") if device_product_name: device_json.update({"device_product": device_product_name}) device_data.update({device_id: device_json}) fabric_refs = result.get('fabric_refs') if fabric_refs and len(fabric_refs) > 0: fabric_fq_name = result.get('fabric_refs')[0].get('to') fabric_fq_name_str = ':'.join(map(str, fabric_fq_name)) request_params['fabric_fq_name'] = fabric_fq_name_str if len(device_data) > 0: request_params.update({"device_json": device_data})
def run_playbook_process(self, playbook_info): playbook_process = None try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" playbook_process = subprocess32.Popen([ "python", playbook_exec_path, "-i", json.dumps(playbook_info) ], close_fds=True, cwd='/', stdout=subprocess32.PIPE) marked_output = {} markers = [DEVICE_DATA, PLAYBOOK_OUTPUT] while True: output = playbook_process.stdout.readline() if output == '' and playbook_process.poll() is not None: break if output: self._logger.debug(output) # read device list data and store in variable result handler for marker in markers: if marker in output: marked_output[marker] = output marked_jsons = self._extract_marked_json(marked_output) self._device_data = marked_jsons.get(DEVICE_DATA) self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed): playbook_output = None try: # create job log to capture the start of the playbook device_name = \ playbook_info['extra_vars']['playbook_input'].get( 'device_fqname') if device_name: device_name = device_name[-1] playbook_name = playbook_info['uri'].split('/')[-1] msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value, device_name=device_name) if not os.path.exists(playbook_info['uri']): msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id) # Run playbook in a separate process. This is needed since # ansible cannot be used in a greenlet based patched environment playbook_output = self.run_playbook_process( playbook_info, percentage_completed) # create job log to capture completion of the playbook execution msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value, device_name=device_name) return playbook_output except JobException: raise except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed): try: # create job log to capture the start of the playbook device_id = \ playbook_info['extra_vars']['playbook_input'].get( 'device_id', "") msg = MsgBundle.getMessage(MsgBundle.START_EXECUTE_PLAYBOOK_MSG, playbook_uri=playbook_info['uri'], device_id=device_id, input_params=json.dumps( playbook_info['extra_vars'] ['playbook_input'] ['input']), extra_params=json.dumps( playbook_info['extra_vars'] ['playbook_input'] ['params'])) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value) if not os.path.exists(playbook_info['uri']): msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id) # Run playbook in a separate process. This is needed since # ansible cannot be used in a greenlet based patched environment self.run_playbook_process(playbook_info, percentage_completed) # create job log to capture completion of the playbook execution msg = MsgBundle.getMessage(MsgBundle.PB_EXEC_COMPLETE_WITH_INFO, playbook_uri=playbook_info['uri'], device_id=device_id) self._logger.debug(msg) self._job_log_utils.send_job_log(self._job_template.fq_name, self._execution_id, self._fabric_fq_name, msg, JobStatus.IN_PROGRESS.value) except JobException: raise except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id)
def send_job_log(self, job_template_fqname, job_execution_id, fabric_fq_name, message, status, completion_percent=None, result=None, timestamp=None, device_name=None, details=None): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) if timestamp is None: timestamp = int(round(time.time() * 1000)) details_str = json.dumps(details) if details else None job_log_entry = JobLogEntry( name=job_template_fqname, execution_id=job_execution_id, fabric_name=fabric_fq_name, timestamp=timestamp, message=message, status=status, percentage_completed=completion_percent, result=result, device_name=device_name, details=details_str) job_log = JobLog(log_entry=job_log_entry) job_log.send(sandesh=self.config_logger._sandesh) self.config_logger.debug("Created job log for job template: %s, " " execution id: %s, fabric_fq_name: %s" "status: %s, completion_percent %s, " "result: " "%s, message: %s" % (job_template_fqname, job_execution_id, fabric_fq_name, status, completion_percent, result, message)) except Exception as e: msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_LOG_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, fabric_name=fabric_fq_name, exc_msg=repr(e)) raise JobException(msg, job_execution_id)
def _validate_job_input(self, input_schema, ip_json): if ip_json is None: msg = MsgBundle.getMessage(MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND) raise JobException(msg, self.job_execution_id) try: ip_schema_json = input_schema if isinstance(input_schema, str): ip_schema_json = json.loads(input_schema) jsonschema.validate(ip_json, ip_schema_json) self._logger.debug("Input Schema Validation Successful" "for template %s" % self.job_template_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA, job_template_id=self.job_template_id, exc_obj=exp) raise JobException(msg, self.job_execution_id)
def send_prouter_job_uve(self, job_template_fqname, fq_names, job_execution_id, prouter_state=None, job_status=None): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) if prouter_state is None: prouter_job_data = PhysicalRouterJobExecution( name=fq_names, execution_id=job_execution_id, job_status=job_status) prouter_job_data = PhysicalRouterJobExecution( name=fq_names, execution_id=job_execution_id, prouter_state=prouter_state, job_status=job_status) prouter_job_uve = PhysicalRouterJobUve( data=prouter_job_data, sandesh=self.config_logger._sandesh) prouter_job_uve.send(sandesh=self.config_logger._sandesh) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def initialize_sandesh_logger(self, config_args, sandesh=True, sandesh_instance=None): # parse the logger args args = self.parse_logger_args(config_args) args.random_collectors = args.collectors if args.collectors: args.random_collectors = random.sample(args.collectors, len(args.collectors)) self.args = args # initialize logger logger = JobLogger(args=args, sandesh_instance_id=self.sandesh_instance_id, sandesh_instance=sandesh_instance) if not sandesh_instance and sandesh: try: sandesh_util = SandeshUtils(logger) sandesh_util.wait_for_connection_establish() except JobException: msg = MsgBundle.getMessage( MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR) raise JobException(msg) logger.info("Sandesh is initialized." " Config logger instance created.") return logger
def start_job(self): # spawn job greenlets job_handler = JobHandler(self._logger, self._vnc_api, self.job_template, self.job_execution_id, self.job_data, self.job_utils, self.device_json, self.auth_token, self.contrail_cluster_id, self.api_server_host, self.job_log_utils, self.sandesh_args, self.fabric_fq_name, self.job_log_utils.args.playbook_timeout, self.playbook_seq, self.vnc_api_init_params, self._zk_client) self.job_handler = job_handler # check if its a multi device playbook playbooks = self.job_template.get_job_template_playbooks() play_info = playbooks.playbook_info[self.playbook_seq] is_multi_device_playbook = play_info.multi_device_playbook # for fabric config push as part of delete workflow, # device json is not needed. There will be no performance # impact as fabric delete from DM will always have one prouter # uuid in the device_list. if is_multi_device_playbook: if self.device_json is None or not self.device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self.job_execution_id) else: self.handle_multi_device_job(job_handler, self.result_handler) else: self.handle_single_job(job_handler, self.result_handler)
def send_job_execution_uve(self, fabric_fq_name, job_template_fqname, job_execution_id, timestamp=None, percentage_completed=None): try: fabric_job_name = list(job_template_fqname) fabric_job_name.insert(0, fabric_fq_name) fabric_job_uve_name = ':'.join(map(str, fabric_job_name)) job_exe_data = FabricJobExecution( name=fabric_job_uve_name, job_status='IN_PROGRESS', percentage_completed=percentage_completed) job_uve = FabricJobUve(data=job_exe_data, sandesh=self.config_logger._sandesh) job_uve.send(sandesh=self.config_logger._sandesh) except Exception as exp: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def get_job_concurrency(self, job_template_id, job_exec_id): (ok, result) = self.db_read("job-template", job_template_id, ['job_template_concurrency_level']) if not ok: msg = "Error while reading the job concurrency " \ "from the job template with id %s : %s" %\ (job_template_id, result) raise JobException(msg, job_exec_id) return result.get('job_template_concurrency_level')
def test_sandesh_timeout(self): mocked_sandesh_utils = flexmock() flexmock(SandeshUtils, __new__=mocked_sandesh_utils) mocked_sandesh_utils.should_receive('__init__') mocked_sandesh_utils.should_receive('wait_for_connection_establish')\ .and_raise(JobException()) args = {"collectors": ['127.0.0.1:8086']} exc_msg = self.assertRaises(JobException, JobLogUtils, "rdm_exc_id", json.dumps(args)) self.assertEqual( str(exc_msg), "JobException in execution" + " (None): " + MsgBundle.getMessage( MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR))
def run_playbook(self, playbook_info, percentage_completed): try: # create job log to capture the start of the playbook device_name = \ playbook_info['extra_vars']['playbook_input'].get( 'device_fqname') if device_name: device_name = device_name[-1] playbook_name = playbook_info['uri'].split('/')[-1] msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) if not os.path.exists(playbook_info['uri']): msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id) # Run playbook in a separate process. This is needed since # ansible cannot be used in a greenlet based patched environment playbook_output = self.run_playbook_process(playbook_info, percentage_completed) # create job log to capture completion of the playbook execution msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG, playbook_name=playbook_name) self._logger.debug(msg) return playbook_output except JobException: raise except Exception as exp: trace = traceback.format_exc() msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException("%s\n%s" % (msg, trace), self._execution_id)
def start_job(self): # spawn job greenlets job_handler = JobHandler( self._logger, self._vnc_api, self.job_template, self.job_execution_id, self.job_data, self.job_utils, self.device_json, self.auth_token, self.api_server_host, self.job_log_utils, self.sandesh_args, self.fabric_fq_name, self.job_log_utils.args.playbook_timeout, self.playbook_seq) if self.device_json is not None: if not self.device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self.job_execution_id) else: self.handle_multi_device_job(job_handler, self.result_handler) else: self.handle_single_job(job_handler, self.result_handler)
def send_prouter_object_log(self, prouter_fqname, job_execution_id, job_input, job_template_fqname, onboarding_state, os_version=None, serial_num=None, timestamp=None): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) prouter_fqname = self.get_fq_name_log_str(prouter_fqname) if timestamp is None: timestamp = int(round(time.time() * 1000)) # create the prouter object log prouter_log_entry = PRouterOnboardingLogEntry( name=prouter_fqname, job_execution_id=job_execution_id, os_version=os_version, serial_num=serial_num, onboarding_state=onboarding_state, timestamp=timestamp, job_template_fqname=job_template_fqname, job_input=job_input) prouter_log = PRouterOnboardingLog(log_entry=prouter_log_entry) prouter_log.send(sandesh=self.config_logger._sandesh) self.config_logger.debug( "Created prouter object log for router: %s, " " execution id: %s, job_template: %s, os_version: " "%s, serial_num: %s, onboarding_state %s" % (prouter_fqname, job_execution_id, job_template_fqname, os_version, serial_num, onboarding_state)) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.SEND_PROUTER_OBJECT_LOG_ERROR, prouter_fqname=prouter_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
def send_job_execution_uve(self, job_template_fqname, job_execution_id, timestamp=None, percentage_completed=None): try: job_template_fqname = self.get_fq_name_log_str(job_template_fqname) if timestamp is None: timestamp = int(round(time.time() * 1000)) job_exe_data = JobExecution( name=job_template_fqname, execution_id=job_execution_id, job_start_ts=timestamp, percentage_completed=percentage_completed) job_uve = UveJobExecution(data=job_exe_data) job_uve.send(sandesh=self.config_logger._sandesh) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR, job_template_fqname=job_template_fqname, job_execution_id=job_execution_id, exc_msg=repr(exp)) raise JobException(msg, job_execution_id)
self._logger.info("Device data json: " + str(self._device_data)) playbook_process.wait(timeout=self._playbook_timeout) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) raise JobException(msg, self._execution_id) def run_playbook(self, playbook_info): try: # create job log to capture the start of the playbook device_id = \ playbook_info['extra_vars']['playbook_input'].get( 'device_id', "") msg = MsgBundle.getMessage( MsgBundle.START_EXECUTE_PLAYBOOK_MSG, playbook_uri=playbook_info['uri'],
def read_device_data(self, device_list, request_params, job_exec_id, is_delete=False): device_data = dict() for device_id in device_list: if not is_delete: try: (ok, result) = self.db_read("physical-router", device_id, [ 'physical_router_user_credentials', 'physical_router_management_ip', 'fq_name', 'physical_router_device_family', 'physical_router_vendor_name', 'physical_router_product_name', 'fabric_refs' ]) if not ok: msg = "Error while reading the physical router " \ "with id %s : %s" % (device_id, result) raise JobException(msg, job_exec_id) except NoIdError as ex: msg = "Device not found" \ "%s: %s" % (device_id, str(ex)) raise JobException(msg, job_exec_id) except Exception as e: msg = "Exception while reading device %s %s " % \ (device_id, str(e)) raise JobException(msg, job_exec_id) device_fq_name = result.get('fq_name') device_mgmt_ip = result.get('physical_router_management_ip') user_cred = result.get('physical_router_user_credentials') device_family = result.get("physical_router_device_family") device_vendor_name = result.get("physical_router_vendor_name") device_product_name = result.get( "physical_router_product_name") fabric_refs = result.get('fabric_refs') if fabric_refs: fabric_fq_name = result.get('fabric_refs')[0].get('to') fabric_fq_name_str = ':'.join(fabric_fq_name) request_params['fabric_fq_name'] = fabric_fq_name_str else: device_mgmt_ip = request_params.get( 'input', {}).get('device_management_ip') device_abs_cfg = request_params.get( 'input', {}).get('device_abstract_config') system = device_abs_cfg.get('system', {}) device_name = system.get('name') device_username = system.get('credentials', {}).get('user_name') device_password = system.get('credentials', {}).get('password') user_cred = { "username": device_username, "password": device_password } device_family = system.get('device_family') device_vendor_name = system.get('vendor_name') device_product_name = system.get('product_name') device_fq_name = ["default-global-system-config", device_name] self.read_fabric_data(request_params, job_exec_id, is_delete) device_json = {"device_management_ip": device_mgmt_ip} device_json.update({"device_fqname": device_fq_name}) if user_cred: device_json.update( {"device_username": user_cred.get('username')}) device_json.update( {"device_password": user_cred.get('password')}) if device_family: device_json.update({"device_family": device_family}) if device_vendor_name: device_json.update({"device_vendor": device_vendor_name}) if device_product_name: device_json.update({"device_product": device_product_name}) device_data.update({device_id: device_json}) if len(device_data) > 0: request_params.update({"device_json": device_data})
def get_playbook_info(self, job_percent_per_task, device_id=None): try: # create the cmd line param for the playbook extra_vars = { 'input': self._job_input, 'prev_pb_output': self._playbook_output, 'params': self._job_params, 'job_template_id': self._job_template.get_uuid(), 'job_template_fqname': self._job_template.fq_name, 'fabric_fq_name': self._fabric_fq_name, 'auth_token': self._auth_token, 'job_execution_id': self._execution_id, 'args': self._sandesh_args, 'playbook_job_percentage': job_percent_per_task } playbooks = self._job_template.get_job_template_playbooks() if device_id: if not self._device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self._execution_id) device_data = self._device_json.get(device_id) if not device_data: msg = MsgBundle.getMessage(MsgBundle.NO_DEVICE_DATA_FOUND, device_id=device_id) raise JobException(msg, self._execution_id) device_family = device_data.get('device_family') device_vendor = device_data.get('device_vendor') if not device_vendor or not device_family: msg = MsgBundle.getMessage( MsgBundle.DEVICE_VENDOR_FAMILY_MISSING, device_id=device_id) raise JobException(msg, self._execution_id) # check for credentials,required param; else playbooks # will fail device_username = device_data.get('device_username') device_password = device_data.get('device_password') if not device_username or not device_password: msg = MsgBundle.getMessage(MsgBundle.NO_CREDENTIALS_FOUND, device_id=device_id) raise JobException(msg, self._execution_id) # update extra-vars to reflect device-related params device_fqname = device_data.get('device_fqname') device_management_ip = device_data.get('device_management_ip') extra_vars.update({ 'device_id': device_id, 'device_fqname': device_fqname, 'device_management_ip': device_management_ip, 'vendor': device_vendor, 'device_family': device_family, 'device_username': device_username, 'device_password': device_password }) self._logger.debug("Passing the following device " "ip to playbook %s " % device_management_ip) # get the playbook uri from the job template play_info = playbooks.playbook_info[self._playbook_seq] playbook_input = {'playbook_input': extra_vars} playbook_info = dict() playbook_info['uri'] = play_info.playbook_uri playbook_info['extra_vars'] = playbook_input return playbook_info except JobException: raise except Exception as exp: msg = MsgBundle.getMessage( MsgBundle.GET_PLAYBOOK_INFO_ERROR, job_template_id=self._job_template.get_uuid(), exc_msg=repr(exp)) raise JobException(msg, self._execution_id)
def run_playbook_process(self, playbook_info, percentage_completed): playbook_process = None playbook_output = None pr_uve_name = None self.current_percentage = percentage_completed try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" unique_pb_id = str(uuid.uuid4()) playbook_info['extra_vars']['playbook_input']['unique_pb_id']\ = unique_pb_id exec_id =\ playbook_info['extra_vars']['playbook_input'][ 'job_execution_id'] device_fqname = \ playbook_info['extra_vars']['playbook_input'].get( 'device_fqname') if device_fqname: pr_fqname = ':'.join(map(str, device_fqname)) job_template_fq_name = ':'.join( map(str, self._job_template.fq_name)) pr_uve_name = pr_fqname + ":" + \ self._fabric_fq_name + ":" + job_template_fq_name pr_object_log_start_time = time.time() playbook_process = subprocess32.Popen([ "python", playbook_exec_path, "-i", json.dumps(playbook_info) ], close_fds=True, cwd='/') # this is to yield the context to the playbooks so that # they start running concurrently gevent.sleep(0) marked_output = self.process_file_and_get_marked_output( unique_pb_id, exec_id, playbook_process, pr_uve_name) marked_jsons = self._extract_marked_json(marked_output) playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT) playbook_process.wait(timeout=self._playbook_timeout) pr_object_log_end_time = time.time() # create prouter UVE in job_manager only if it is not a multi # device job template if not self._job_template.get_job_template_multi_device_job(): self.send_prouter_uve(exec_id, pr_object_log_start_time, pr_object_log_end_time, playbook_process.returncode) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id) except Exception as exp: msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR, playbook_uri=playbook_info['uri'], exc_msg=repr(exp)) raise JobException(msg, self._execution_id) if playbook_process.returncode != 0: msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR, playbook_uri=playbook_info['uri']) if playbook_output: msg = msg + "\n Error Message from playbook: %s" % playbook_output.get( 'message', "") raise JobException(msg, self._execution_id) return playbook_output
def start_job(self): self._logger.info("Starting Executable") job_error_msg = None job_template = self.job_template try: # create job UVE and log self.result_handler = JobResultHandler(self.job_template_id, self.job_execution_id, self.fabric_fq_name, self._logger, self.job_utils, self.job_log_utils) msg = MsgBundle.getMessage(MsgBundle.START_JOB_MESSAGE, job_execution_id=self.job_execution_id, job_template_name=\ job_template.fq_name[-1]) self._logger.debug(msg) timestamp = int(round(time.time() * 1000)) self.job_log_utils.send_job_log(job_template.fq_name, self.job_execution_id, self.fabric_fq_name, msg, JobStatus.STARTING.value, timestamp=timestamp) # validate job input if required by job_template input_schema input_schema = job_template.get_job_template_input_schema() if input_schema: self._validate_job_input(input_schema, self.job_data) executable_list = job_template.get_job_template_executables()\ .get_executable_info() for executable in executable_list: exec_path = executable.get_executable_path() exec_args = executable.get_executable_args() job_input_args = self.gather_job_args() try: exec_process = subprocess32.Popen([exec_path, "--job-input", json.dumps(job_input_args), '--debug', 'True'], close_fds=True, cwd='/', stdout=subprocess32.PIPE, stderr=subprocess32.PIPE) self.job_file_write.write_to_file( self.job_execution_id, "job_summary", JobFileWrite.JOB_LOG, {"job_status": "INPROGRESS"}) msg = "Child process pid = " + str(exec_process.pid) self._logger.info(msg) (out, err) = exec_process.communicate(timeout=self.executable_timeout) self._logger.notice(str(out)) self._logger.notice(str(err)) except subprocess32.TimeoutExpired as timeout_exp: if exec_process is not None: os.kill(exec_process.pid, 9) msg = MsgBundle.getMessage( MsgBundle.RUN_EXECUTABLE_PROCESS_TIMEOUT, exec_path=exec_path, exc_msg=repr(timeout_exp)) raise JobException(msg, self.job_execution_id) self._logger.info(exec_process.returncode) self._logger.info("Executable Completed") if exec_process.returncode != 0: self.job_file_write.write_to_file( self.job_execution_id, "job_summary", JobFileWrite.JOB_LOG, {"job_status": "FAILED"}) msg = MsgBundle.getMessage(MsgBundle. EXECUTABLE_RETURN_WITH_ERROR, exec_uri=exec_path) self._logger.error(msg) else: self.job_file_write.write_to_file( self.job_execution_id, "job_summary", JobFileWrite.JOB_LOG, {"job_status": "COMPLETED"}) except JobException as exp: err_msg = "Job Exception recieved: %s " % repr(exp) self._logger.error(err_msg) self._logger.error("%s" % traceback.format_exc()) self.result_handler.update_job_status(JobStatus.FAILURE, err_msg) if job_template: self.result_handler.create_job_summary_log( job_template.fq_name) job_error_msg = err_msg except Exception as exp: err_msg = "Error while executing job %s " % repr(exp) self._logger.error(err_msg) self._logger.error("%s" % traceback.format_exc()) self.result_handler.update_job_status(JobStatus.FAILURE, err_msg) self.result_handler.create_job_summary_log(job_template.fq_name) job_error_msg = err_msg finally: # need to wait for the last job log and uve update to complete # via sandesh and then close sandesh connection sandesh_util = SandeshUtils(self._logger) sandesh_util.close_sandesh_connection() self._logger.info("Closed Sandesh connection") if job_error_msg is not None: sys.exit(job_error_msg)
def handle_job(self, result_handler, job_percent_per_task, device_id=None, device_name=None): playbook_info = None try: msg = "Starting playbook execution for job template %s with " \ "execution id %s" % (self._job_template.get_uuid(), self._execution_id) self._logger.debug(msg) # Always acquire the lock while executing the multi device jobs if device_name is not None: if not self._acquire_device_lock(device_name): raise JobException( MsgBundle.getMessage(MsgBundle.DEVICE_LOCK_FAILURE)) # get the playbook information from the job template playbook_info = self.get_playbook_info(job_percent_per_task, device_id) # run the playbook and retrieve the playbook output if any playbook_output = self.run_playbook( playbook_info, result_handler.percentage_completed) # retrieve the device_op_results in case it was set for # generic device operations. # retrieve the playbook output status for warning cases playbook_output_results = None playbook_output_status = "" warning_msg = "" if playbook_output: playbook_output_results = playbook_output.get('results') playbook_output_status = playbook_output.get('status') result_handler.update_playbook_output(playbook_output) warning_msg = playbook_output.get("message") msg = MsgBundle.getMessage( MsgBundle.PLAYBOOK_EXECUTION_COMPLETE, job_template_name=self._job_template.get_fq_name()[-1], job_execution_id=self._execution_id) self._logger.debug(msg) if playbook_output_status.lower() == "warning": result_handler.update_job_status( JobStatus.WARNING, warning_msg, device_id, device_name, pb_results=playbook_output_results) else: result_handler.update_job_status( JobStatus.SUCCESS, msg, device_id, device_name, pb_results=playbook_output_results) self.check_and_send_prouter_job_uve_for_multidevice( playbook_info, JobStatus.SUCCESS.value, playbook_output_results) if self.current_percentage: result_handler.percentage_completed = self.current_percentage except JobException as job_exp: self._logger.error("%s" % job_exp.msg) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg, device_id, device_name) if playbook_info: self.check_and_send_prouter_job_uve_for_multidevice( playbook_info, JobStatus.FAILURE.value) except Exception as exp: self._logger.error("Error while executing job %s " % repr(exp)) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, exp.message, device_id, device_name) if playbook_info: self.check_and_send_prouter_job_uve_for_multidevice( playbook_info, JobStatus.FAILURE.value) finally: if device_name is not None: self._release_device_lock(device_name)
class JobHandler(object): def __init__(self, logger, vnc_api, job_template, execution_id, input, params, job_utils, device_json, auth_token, job_log_utils, sandesh_args, fabric_fq_name, playbook_timeout, playbook_seq): self._logger = logger self._vnc_api = vnc_api self._job_template = job_template self._execution_id = execution_id self._job_input = input self._job_params = params self._job_utils = job_utils self._device_json = device_json self._auth_token = auth_token self._job_log_utils = job_log_utils self._sandesh_args = sandesh_args self._fabric_fq_name = fabric_fq_name self._playbook_timeout = playbook_timeout self._playbook_seq = playbook_seq self._device_data = None def handle_job(self, result_handler, job_percent_per_task, device_id=None): try: msg = "Starting playbook execution for job template %s with " \ "execution id %s" % (self._job_template.get_uuid(), self._execution_id) self._logger.debug(msg) # get the playbook information from the job template playbook_info = self.get_playbook_info(job_percent_per_task, device_id) # run the playbook self.run_playbook(playbook_info) msg = MsgBundle.getMessage( MsgBundle.PLAYBOOK_EXECUTION_COMPLETE, job_template_id=self._job_template.get_uuid(), job_execution_id=self._execution_id) self._logger.debug(msg) result_handler.update_job_status(JobStatus.SUCCESS, msg, device_id) self.update_result_handler_device_data(result_handler) except JobException as job_exp: self._logger.error("%s" % job_exp.msg) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg, device_id) except Exception as exp: self._logger.error("Error while executing job %s " % repr(exp)) self._logger.error("%s" % traceback.format_exc()) result_handler.update_job_status(JobStatus.FAILURE, exp.message, device_id) # def find_playbook_info(self, device_family, device_vendor, playbook_list): # # for playbook_info in playbook_list: # pb_vendor_name = playbook_info.get_vendor() # if not pb_vendor_name: # # device_vendor agnostic # return playbook_info # if pb_vendor_name.lower() == device_vendor.lower(): # pb_device_family = playbook_info.get_device_family() # if pb_device_family: # if device_family.lower() == pb_device_family.lower(): # return playbook_info # else: # # device_family agnostic # return playbook_info # msg = MsgBundle.getMessage(MsgBundle. # PLAYBOOK_INFO_DEVICE_MISMATCH, # device_vendor=device_vendor, # device_family=device_family) # raise JobException(msg, self._execution_id) def get_playbook_info(self, job_percent_per_task, device_id=None): try: # create the cmd line param for the playbook extra_vars = { 'input': self._job_input, 'params': self._job_params, 'job_template_id': self._job_template.get_uuid(), 'job_template_fqname': self._job_template.fq_name, 'fabric_fq_name': self._fabric_fq_name, 'auth_token': self._auth_token, 'job_execution_id': self._execution_id, 'args': self._sandesh_args, 'playbook_job_percentage': job_percent_per_task } playbooks = self._job_template.get_job_template_playbooks() if device_id: if not self._device_json: msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) raise JobException(msg, self._execution_id) device_data = self._device_json.get(device_id) if not device_data: msg = MsgBundle.getMessage(MsgBundle.NO_DEVICE_DATA_FOUND, device_id=device_id) raise JobException(msg, self._execution_id) device_family = device_data.get('device_family') device_vendor = device_data.get('device_vendor') if not device_vendor or not device_family: msg = MsgBundle.getMessage( MsgBundle.DEVICE_VENDOR_FAMILY_MISSING, device_id=device_id) raise JobException(msg, self._execution_id) # check for credentials,required param; else playbooks # will fail device_username = device_data.get('device_username') device_password = device_data.get('device_password') if not device_username or not device_password: msg = MsgBundle.getMessage(MsgBundle.NO_CREDENTIALS_FOUND, device_id=device_id) raise JobException(msg, self._execution_id) # update extra-vars to reflect device-related params device_fqname = device_data.get('device_fqname') device_management_ip = device_data.get('device_management_ip') extra_vars.update({ 'device_id': device_id, 'device_fqname': device_fqname, 'device_management_ip': device_management_ip, 'vendor': device_vendor, 'device_family': device_family, 'device_username': device_username, 'device_password': device_password }) self._logger.debug("Passing the following device " "ip to playbook %s " % device_management_ip) # get the playbook uri from the job template play_info = playbooks.playbook_info[self._playbook_seq] playbook_input = {'playbook_input': extra_vars} playbook_info = dict() playbook_info['uri'] = play_info.playbook_uri playbook_info['extra_vars'] = playbook_input return playbook_info except JobException: raise except Exception as exp: msg = MsgBundle.getMessage( MsgBundle.GET_PLAYBOOK_INFO_ERROR, job_template_id=self._job_template.get_uuid(), exc_msg=repr(exp)) raise JobException(msg, self._execution_id) def run_playbook_process(self, playbook_info): playbook_process = None try: playbook_exec_path = os.path.dirname(__file__) \ + "/playbook_helper.py" playbook_process = subprocess32.Popen([ "python", playbook_exec_path, "-i", json.dumps(playbook_info) ], close_fds=True, cwd='/', stdout=subprocess32.PIPE) device_data_output = "" device_data_keyword = 'DEVICEDATA##' while True: output = playbook_process.stdout.readline() if output == '' and playbook_process.poll() is not None: break if output: self._logger.debug(output) # read device list data and store in variable result handler if device_data_keyword in output: device_data_output = output if device_data_output != "": device_data_output = self._extract_device_data_output( device_data_output, device_data_keyword) self._logger.info("Device data extracted from output" + str(device_data_output)) try: self._device_data = json.loads(device_data_output) except ValueError, e: self._device_data = ast.literal_eval(device_data_output) self._logger.info("Device data json: " + str(self._device_data)) playbook_process.wait(timeout=self._playbook_timeout) except subprocess32.TimeoutExpired as timeout_exp: if playbook_process is not None: os.kill(playbook_process.pid, 9) msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=playbook_info['uri'], exc_msg=repr(timeout_exp)) raise JobException(msg, self._execution_id)