def test_execute_job_no_device_json(self):
# create job template
job_template = JobTemplate(
job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=True,
job_template_playbooks=TestJobManagerUtils.playbooks_list,
name='multi_device_no_json')
job_template_uuid = self._vnc_lib.job_template_create(job_template)
device_id = "aad74e24-a00b-4eb3-8412-f8b9412925c3"
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
job_template_uuid)
job_input_json.update({
"params": {
"device_list": [device_id]
},
})
jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib,
job_input_json, log_utils)
sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job)
self.assertEqual(
sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) + device_id +
",\n" + MsgBundle.getMessage(MsgBundle.PLAYBOOK_RESULTS_MESSAGE) +
device_id + ":" +
MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND) + " \n")
def test_execute_job_run_pb_gen_exc(self):
# create job template
job_template = JobTemplate(
job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=False,
job_template_playbooks=TestJobManagerUtils.playbooks_list,
name='run_pb_gen_exc')
job_template_uuid = self._vnc_lib.job_template_create(job_template)
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
job_template_uuid)
jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib,
job_input_json, log_utils)
exc = Exception('some gen exc')
# mock the call to raise exception
flexmock(json).should_receive('dumps')\
.and_raise(exc)
sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job)
self.assertEqual(
sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.RUN_PLAYBOOK_ERROR,
playbook_uri=TestJobManagerUtils.play_info.playbook_uri,
exc_msg=repr(exc)))
def test_execute_job_no_pb_file_on_path(self):
# create job template
play_info = PlaybookInfoType(playbook_uri='job_manager_test.yaml')
playbooks_list = PlaybookInfoListType(playbook_info=[play_info])
job_template = JobTemplate(
job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=False,
job_template_playbooks=TestJobManagerUtils.playbooks_list,
name='single_device_template')
job_template_uuid = self._vnc_lib.job_template_create(job_template)
playbook_info = {"uri": "job_manager_test.yaml"}
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
job_template_uuid)
jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib,
job_input_json, log_utils)
# mock the call to invoke the playbook process
flexmock(os.path).should_receive('exists').and_return(False)
sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job)
self.assertEqual(
sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) +
MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND,
playbook_uri=playbook_info['uri']))
def run_playbook_process(self, playbook_info, percentage_completed):
playbook_process = None
self.current_percentage = percentage_completed
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
unique_pb_id = str(uuid.uuid4())
playbook_info['extra_vars']['playbook_input']['unique_pb_id']\
= unique_pb_id
exec_id =\
playbook_info['extra_vars']['playbook_input'][
'job_execution_id']
pr_object_log_start_time = time.time()
playbook_process = subprocess32.Popen(["python",
playbook_exec_path,
"-i",
json.dumps(playbook_info)],
close_fds=True, cwd='/')
# this is to yield the context to the playbooks so that
# they start running concurrently
time.sleep(0.5)
marked_output = self.process_file_and_get_marked_output(
unique_pb_id, exec_id, playbook_process
)
marked_jsons = self._extract_marked_json(marked_output)
self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
pr_object_log_end_time = time.time()
self.send_pr_object_log(
exec_id,
pr_object_log_start_time,
pr_object_log_end_time,
playbook_process.returncode)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(
MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.
RUN_PLAYBOOK_PROCESS_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
raise JobException(msg, self._execution_id)
def create_job_summary_message(self):
job_summary_message = MsgBundle.getMessage(
MsgBundle.JOB_SUMMARY_MESSAGE_HDR)
if self.job_result_status is None:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_RESULT_STATUS_NONE)
elif self.job_result_status == JobStatus.FAILURE:
if len(self.failed_device_jobs) > 0:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR)
for failed_device in self.failed_device_jobs:
msg = failed_device + ','
job_summary_message += msg
else:
job_summary_message += "Job failed. "
job_summary_message += "\n"
elif self.job_result_status == JobStatus.SUCCESS:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_EXECUTION_COMPLETE)
if len(self.job_result) > 0:
job_summary_message += MsgBundle.getMessage(
MsgBundle.PLAYBOOK_RESULTS_MESSAGE)
result_summary = ""
for entry in self.job_result:
result_summary += \
"%s:%s \n" % (entry, self.job_result[entry])
job_summary_message += result_summary
if self.job_result_message is not None:
job_summary_message += self.job_result_message
return job_summary_message
def run_playbook_process(self, playbook_info, percentage_completed):
playbook_process = None
self.current_percentage = percentage_completed
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
unique_pb_id = str(uuid.uuid4())
playbook_info['extra_vars']['playbook_input']['unique_pb_id']\
= unique_pb_id
exec_id =\
playbook_info['extra_vars']['playbook_input'][
'job_execution_id']
pr_uve_name = self.get_pr_uve_name_from_device_name(playbook_info)
playbook_process = subprocess32.Popen(["python",
playbook_exec_path,
"-i",
json.dumps(playbook_info)],
close_fds=True, cwd='/')
# Save process ID in case of abort
self._pb_pids.append(playbook_process.pid)
# this is to yield the context to the playbooks so that
# they start running concurrently
gevent.sleep(0)
marked_output = self.process_file_and_get_marked_output(
unique_pb_id, exec_id, playbook_process, pr_uve_name)
marked_jsons = self._extract_marked_json(marked_output)
playbook_output = marked_jsons.get(JobFileWrite.PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
# create prouter UVE in job_manager only if it is not a multi
# device job template
if not self.is_multi_device_playbook:
status = "SUCCESS" if playbook_process.returncode == 0 \
else "FAILURE"
self.send_prouter_uve(exec_id, status)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(
MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
if playbook_output:
msg = "%s\n Error Message from playbook: %s" % (
msg, playbook_output.get('message', "")
)
raise JobException(msg, self._execution_id)
return playbook_output
def test_execute_job_run_pb_process_rc1(self):
# create job template
job_template = JobTemplate(job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=False,
job_template_playbooks=TestJobManagerUtils.
playbooks_list,
name='run_pb_prc_rc_1')
job_template_uuid = self._vnc_lib.job_template_create(job_template)
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
job_template_uuid)
wm = WFManager(log_utils.get_config_logger(),
self._vnc_lib, job_input_json, log_utils)
loop_var = 0
def mock_readline():
global loop_var
loop_var += 1
if loop_var == 1:
with open("tests/test.txt", 'r') as f:
line = f.readline()
return line
if loop_var == 2:
fake_process.should_receive("poll").and_return(123)
loop_var = 0
return ""
stdout = flexmock(readline=mock_readline)
flexmock(json).should_receive("loads")
flexmock(os.path).should_receive('exists').and_return(True)
# mock the call to raise exception
fake_resp = flexmock(status_code=123)
fake_request = flexmock(requests).should_receive(
'post').and_return(fake_resp)
fake_process = flexmock(returncode=1, stdout=stdout)
fake_process.should_receive('wait')
fake_process.should_receive('poll').and_return(123)
# flexmock(subprocess).should_receive('TimeoutExpired')
flexmock(subprocess32).should_receive('Popen').and_return(
fake_process)
sys_exit_msg = self.assertRaises(SystemExit, wm.start_job)
self.assertEqual(sys_exit_msg.code,
MsgBundle.getMessage(
MsgBundle.JOB_SUMMARY_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=TestJobManagerUtils.
play_info.playbook_uri)
)
def run_playbook_process(self, playbook_info, percentage_completed):
playbook_process = None
self.current_percentage = percentage_completed
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
unique_pb_id = str(uuid.uuid4())
playbook_info['extra_vars']['playbook_input']['unique_pb_id']\
= unique_pb_id
exec_id =\
playbook_info['extra_vars']['playbook_input'][
'job_execution_id']
pr_uve_name = self.get_pr_uve_name_from_device_name(playbook_info)
playbook_process = subprocess32.Popen(["python",
playbook_exec_path,
"-i",
json.dumps(playbook_info)],
close_fds=True, cwd='/')
# this is to yield the context to the playbooks so that
# they start running concurrently
gevent.sleep(0)
marked_output = self.process_file_and_get_marked_output(
unique_pb_id, exec_id, playbook_process, pr_uve_name)
marked_jsons = self._extract_marked_json(marked_output)
playbook_output = marked_jsons.get(JobFileWrite.PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
# create prouter UVE in job_manager only if it is not a multi
# device job template
if not self._job_template.get_job_template_multi_device_job():
status = "SUCCESS" if playbook_process.returncode == 0 \
else "FAILURE"
self.send_prouter_uve(exec_id, status)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(
MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
if playbook_output:
msg = "%s\n Error Message from playbook: %s" % (
msg, playbook_output.get('message', "")
)
raise JobException(msg, self._execution_id)
return playbook_output
def handle_init_failure(job_input_json, error_num, error_msg):
logger.error(MsgBundle.getMessage(error_num,
exc_msg=traceback.format_exc()))
msg = MsgBundle.getMessage(error_num, exc_msg=error_msg)
job_log_utils.send_job_log(job_input_json.get('job_template_fq_name'),
job_input_json.get('job_execution_id'),
job_input_json.get('fabric_fq_name'),
msg, JobStatus.FAILURE)
sys.exit(msg)
def handle_init_failure(job_input_json, error_num, error_msg):
logger.error(MsgBundle.getMessage(error_num,
exc_msg=traceback.format_exc()))
msg = MsgBundle.getMessage(error_num, exc_msg=error_msg)
job_log_utils.send_job_log(job_input_json.get('job_template_fq_name'),
job_input_json.get('job_execution_id'),
job_input_json.get('fabric_fq_name'),
msg, JobStatus.FAILURE)
sys.exit(msg)
def create_job_summary_message(self):
job_summary_message = MsgBundle.getMessage(
MsgBundle.JOB_SUMMARY_MESSAGE_HDR)
failed_device_jobs_len = len(self.failed_device_jobs)
if self.job_result_status is None:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_RESULT_STATUS_NONE)
elif self.job_result_status == JobStatus.FAILURE:
if failed_device_jobs_len > 0:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR)
for failed_device in self.failed_device_jobs:
msg = failed_device + ','
job_summary_message += msg
else:
job_summary_message += "Job failed. "
job_summary_message += "\n"
elif self.job_result_status == JobStatus.SUCCESS:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_EXECUTION_COMPLETE)
device_job_result_len = len(self.job_result)
if device_job_result_len > 0:
job_summary_message += MsgBundle.getMessage(
MsgBundle.PLAYBOOK_RESULTS_MESSAGE)
job_summary_message += "Successfully completed "\
"job for %s devices.\n"\
% (device_job_result_len -
failed_device_jobs_len)
# result_summary would infact be the failed_devices
# result summary
result_summary = ""
device_op_results = []
failed_device_names = []
for entry in self.job_result:
if entry in self.failed_device_jobs:
result_summary += \
"%s:%s \n" % (self.job_result[entry]['device_name'],
self.job_result[entry]['message'])
failed_device_names.append(
self.job_result[entry]['device_name'])
elif self.job_result[entry]['device_op_result']:
# could be other device jobs such as device import, topology
device_op_results.append(
self.job_result[entry]['device_op_result'])
if result_summary != "":
failed_device_msg = "Job execution failed for %s devices.\n"\
% len(self.failed_device_jobs)
result_summary = failed_device_msg + result_summary
job_summary_message += result_summary
if self.job_result_message is not None:
job_summary_message += self.job_result_message
return job_summary_message, device_op_results, failed_device_names
def create_job_summary_message(self):
job_summary_message = MsgBundle.getMessage(
MsgBundle.JOB_SUMMARY_MESSAGE_HDR)
failed_device_jobs_len = len(self.failed_device_jobs)
if self.job_result_status is None:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_RESULT_STATUS_NONE)
elif self.job_result_status == JobStatus.FAILURE:
if failed_device_jobs_len > 0:
job_summary_message += MsgBundle.getMessage(
MsgBundle.
JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR)
for failed_device in self.failed_device_jobs:
msg = failed_device + ','
job_summary_message += msg
else:
job_summary_message += "Job failed. "
job_summary_message += "\n"
elif self.job_result_status == JobStatus.SUCCESS:
job_summary_message += MsgBundle.getMessage(
MsgBundle.JOB_EXECUTION_COMPLETE)
device_job_result_len = len(self.job_result)
if device_job_result_len > 0:
job_summary_message += MsgBundle.getMessage(
MsgBundle.PLAYBOOK_RESULTS_MESSAGE)
job_summary_message += "Successfully completed "\
"job for %s devices.\n"\
% (device_job_result_len - failed_device_jobs_len)
# result_summary would infact be the failed_devices
# result summary
result_summary = ""
device_op_results = []
failed_device_names = []
for entry in self.job_result:
if entry in self.failed_device_jobs:
result_summary += \
"%s:%s \n" % (self.job_result[entry]['device_name'],
self.job_result[entry]['message'])
failed_device_names.append(
self.job_result[entry]['device_name'])
elif self.job_result[entry]['device_op_result']:
# could be other device jobs such as device import, topology
device_op_results.append(
self.job_result[entry]['device_op_result'])
if result_summary != "":
failed_device_msg = "Job execution failed for %s devices.\n"\
% len(self.failed_device_jobs)
result_summary = failed_device_msg + result_summary
job_summary_message += result_summary
if self.job_result_message is not None:
job_summary_message += self.job_result_message
return job_summary_message, device_op_results, failed_device_names
def test_execute_job_input_schema_ip_not_found(self):
# create job template
fake_job_template = flexmock(
job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=False,
job_template_playbooks=TestJobManagerUtils.
playbooks_list,
name='input_schema_template_ip',
fq_name=["default-global-system-config",
"input_schema_template_ip"],
uuid='random_uuid')
mock_vnc = flexmock()
flexmock(VncApi).new_instances(mock_vnc)
flexmock(mock_vnc).should_receive('job_template_read') \
.with_args(id="random_uuid") \
.and_return(fake_job_template)
TestJobManagerUtils.mock_sandesh_check()
job_input_json = {
"job_template_id": "random_uuid",
"job_execution_id": TestJobManagerUtils.execution_id,
"fabric_fq_name": "Global-system-config:fabric:1",
"auth_token": "6e7d7f87faa54fac96a2a28ec752336a",
"args": TestJobManagerUtils.args
}
log_utils = JobLogUtils(
sandesh_instance_id=TestJobManagerUtils.execution_id,
config_args=json.dumps(TestJobManagerUtils.args))
wm = WFManager(log_utils.get_config_logger(),
mock_vnc, job_input_json, log_utils)
fake_schema = TestJobManagerUtils.fake_schema
fake_job_template.should_receive('get_job_template_input_schema') \
.and_return(fake_schema)
fake_job_template.should_receive(
'get_job_template_multi_device_job') \
.and_return(False)
fake_job_template.should_receive('get_uuid').and_return('random_uuid')
# mock the job_handler to raise an exception
fake_job_template.should_receive('get_job_template_playbooks')
sys_exit_msg = self.assertRaises(SystemExit, wm.start_job)
self.assertEqual(sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_EXC_REC_HDR) +
MsgBundle.getMessage(
MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND) +
" ")
def test_execute_job_handler_gen_exc(self):
# create job template
fake_job_template = flexmock(
job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=False,
job_template_playbooks=TestJobManagerUtils.playbooks_list,
name='single_device_temp_jh_gen_exc',
fq_name=[
"default-global-system-config", "single_device_temp_jh_gen_exc"
],
uuid='random_uuid')
mock_vnc = flexmock()
flexmock(VncApi).new_instances(mock_vnc)
flexmock(mock_vnc).should_receive('job_template_read')\
.with_args(id="random_uuid")\
.and_return(fake_job_template)
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
"random_uuid")
jmgr = JobManager(log_utils.get_config_logger(), mock_vnc,
job_input_json, log_utils)
fake_job_template.should_receive(
'get_job_template_input_schema').and_return(None)
fake_job_template.should_receive(
'get_job_template_multi_device_job').and_return(False)
fake_job_template.should_receive('get_uuid').and_return('random_uuid')
# mock the job_handler to raise a general exception
fake_job_template.should_receive('get_job_template_playbooks')\
.and_raise(
Exception("Mock "
"Generic Exception in job_handler"))
exc = Exception('Mock Generic Exception in job_handler')
sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job)
self.assertEqual(
sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) +
MsgBundle.getMessage(MsgBundle.GET_PLAYBOOK_INFO_ERROR,
job_template_id="random_uuid",
exc_msg=repr(exc)))
def parse_job_input(self, job_input_json):
# job input should have job_template_id and execution_id field
if job_input_json.get('job_template_id') is None:
msg = MsgBundle.getMessage(MsgBundle.JOB_TEMPLATE_MISSING)
raise Exception(msg)
if job_input_json.get('job_execution_id') is None:
msg = MsgBundle.getMessage(MsgBundle.JOB_EXECUTION_ID_MISSING)
raise Exception(msg)
self.job_template_id = job_input_json.get('job_template_id')
self.job_execution_id = job_input_json.get('job_execution_id')
self.job_data = job_input_json.get('input')
self.fabric_fq_name = job_input_json.get('fabric_fq_name')
def run_playbook_process(self, playbook_info):
playbook_process = None
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
playbook_process = subprocess32.Popen([
"python", playbook_exec_path, "-i",
json.dumps(playbook_info)
],
close_fds=True,
cwd='/',
stdout=subprocess32.PIPE)
marked_output = {}
markers = [DEVICE_DATA, PLAYBOOK_OUTPUT]
while True:
output = playbook_process.stdout.readline()
if output == '' and playbook_process.poll() is not None:
break
if output:
self._logger.debug(output)
# read device list data and store in variable result handler
for marker in markers:
if marker in output:
marked_output[marker] = output
marked_jsons = self._extract_marked_json(marked_output)
self._device_data = marked_jsons.get(DEVICE_DATA)
self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed):
try:
# create job log to capture the start of the playbook
device_id = \
playbook_info['extra_vars']['playbook_input'].get(
'device_id', "")
msg = MsgBundle.getMessage(MsgBundle.START_EXECUTE_PLAYBOOK_MSG,
playbook_uri=playbook_info['uri'],
device_id=device_id,
input_params=json.dumps(
playbook_info['extra_vars']
['playbook_input']
['input']),
extra_params=json.dumps(
playbook_info['extra_vars']
['playbook_input']
['params']))
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg, JobStatus.IN_PROGRESS.value)
if not os.path.exists(playbook_info['uri']):
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND,
playbook_uri=playbook_info['uri'])
raise JobException(msg,
self._execution_id)
# Run playbook in a separate process. This is needed since
# ansible cannot be used in a greenlet based patched environment
self.run_playbook_process(playbook_info, percentage_completed)
# create job log to capture completion of the playbook execution
msg = MsgBundle.getMessage(MsgBundle.PB_EXEC_COMPLETE_WITH_INFO,
playbook_uri=playbook_info['uri'],
device_id=device_id)
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg, JobStatus.IN_PROGRESS.value)
except JobException:
raise
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed):
playbook_output = None
try:
# create job log to capture the start of the playbook
device_name = \
playbook_info['extra_vars']['playbook_input'].get(
'device_fqname')
if device_name:
device_name = device_name[-1]
playbook_name = playbook_info['uri'].split('/')[-1]
msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg,
JobStatus.IN_PROGRESS.value,
device_name=device_name)
if not os.path.exists(playbook_info['uri']):
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND,
playbook_uri=playbook_info['uri'])
raise JobException(msg, self._execution_id)
# Run playbook in a separate process. This is needed since
# ansible cannot be used in a greenlet based patched environment
playbook_output = self.run_playbook_process(
playbook_info, percentage_completed)
# create job log to capture completion of the playbook execution
msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg,
JobStatus.IN_PROGRESS.value,
device_name=device_name)
return playbook_output
except JobException:
raise
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
def parse_job_input(self, job_input_json):
# job input should have job_template_id and execution_id field
if job_input_json.get('job_template_id') is None:
msg = MsgBundle.getMessage(MsgBundle.JOB_TEMPLATE_MISSING)
raise Exception(msg)
if job_input_json.get('job_execution_id') is None:
msg = MsgBundle.getMessage(
MsgBundle.JOB_EXECUTION_ID_MISSING)
raise Exception(msg)
self.job_template_id = job_input_json.get('job_template_id')
self.job_execution_id = job_input_json.get('job_execution_id')
self.job_data = job_input_json.get('input')
self.fabric_fq_name = job_input_json.get('fabric_fq_name')
def start_job(self):
# spawn job greenlets
job_handler = JobHandler(self._logger, self._vnc_api,
self.job_template, self.job_execution_id,
self.job_data, self.job_utils,
self.device_json, self.auth_token,
self.contrail_cluster_id,
self.api_server_host, self.job_log_utils,
self.sandesh_args, self.fabric_fq_name,
self.job_log_utils.args.playbook_timeout,
self.playbook_seq, self.vnc_api_init_params,
self._zk_client)
# check if its a multi device playbook
playbooks = self.job_template.get_job_template_playbooks()
play_info = playbooks.playbook_info[self.playbook_seq]
is_multi_device_playbook = play_info.multi_device_playbook
# for fabric config push as part of delete workflow,
# device json is not needed. There will be no performance
# impact as fabric delete from DM will always have one prouter
# uuid in the device_list.
if is_multi_device_playbook:
if self.device_json is None or not self.device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg, self.job_execution_id)
else:
self.handle_multi_device_job(job_handler, self.result_handler)
else:
self.handle_single_job(job_handler, self.result_handler)
def _validate_job_input(self, input_schema, ip_json):
if ip_json is None:
msg = MsgBundle.getMessage(MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND)
raise JobException(msg, self.job_execution_id)
try:
ip_schema_json = input_schema
if isinstance(input_schema, str):
ip_schema_json = json.loads(input_schema)
jsonschema.validate(ip_json, ip_schema_json)
self._logger.debug("Input Schema Validation Successful"
"for template %s" % self.job_template_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA,
job_template_id=self.job_template_id,
exc_obj=exp)
raise JobException(msg, self.job_execution_id)
def handle_job(self, result_handler, job_percent_per_task, device_id=None):
try:
msg = "Starting playbook execution for job template %s with " \
"execution id %s" % (self._job_template.get_uuid(),
self._execution_id)
self._logger.debug(msg)
# get the playbook information from the job template
playbook_info = self.get_playbook_info(job_percent_per_task,
device_id)
# run the playbook
self.run_playbook(playbook_info)
msg = MsgBundle.getMessage(
MsgBundle.PLAYBOOK_EXECUTION_COMPLETE,
job_template_id=self._job_template.get_uuid(),
job_execution_id=self._execution_id)
self._logger.debug(msg)
result_handler.update_job_status(JobStatus.SUCCESS, msg, device_id)
self.update_result_handler(result_handler)
except JobException as job_exp:
self._logger.error("%s" % job_exp.msg)
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg,
device_id)
except Exception as exp:
self._logger.error("Error while executing job %s " % repr(exp))
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, exp.message,
device_id)
def start_job(self):
# spawn job greenlets
job_handler = JobHandler(self._logger, self._vnc_api,
self.job_template, self.job_execution_id,
self.job_data, self.job_utils,
self.device_json, self.auth_token,
self.contrail_cluster_id,
self.api_server_host, self.job_log_utils,
self.sandesh_args, self.fabric_fq_name,
self.job_log_utils.args.playbook_timeout,
self.playbook_seq, self.vnc_api_init_params,
self._zk_client)
self.job_handler = job_handler
# check if its a multi device playbook
playbooks = self.job_template.get_job_template_playbooks()
play_info = playbooks.playbook_info[self.playbook_seq]
is_multi_device_playbook = play_info.multi_device_playbook
# for fabric config push as part of delete workflow,
# device json is not needed. There will be no performance
# impact as fabric delete from DM will always have one prouter
# uuid in the device_list.
if is_multi_device_playbook:
if self.device_json is None or not self.device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg, self.job_execution_id)
else:
self.handle_multi_device_job(job_handler, self.result_handler)
else:
self.handle_single_job(job_handler, self.result_handler)
def send_job_log(self, job_template_fqname, job_execution_id,
fabric_fq_name, message, status, completion_percent=None,
result=None, timestamp=None, device_name=None,
details=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
if timestamp is None:
timestamp = int(round(time.time() * 1000))
details_str = json.dumps(details) if details else None
job_log_entry = JobLogEntry(
name=job_template_fqname, execution_id=job_execution_id,
fabric_name=fabric_fq_name, timestamp=timestamp,
message=message, status=status,
percentage_completed=completion_percent, result=result,
device_name=device_name, details=details_str)
job_log = JobLog(log_entry=job_log_entry)
job_log.send(sandesh=self.config_logger._sandesh)
self.config_logger.debug("Created job log for job template: %s, "
" execution id: %s, fabric_fq_name: %s"
"status: %s, completion_percent %s, "
"result: "
"%s, message: %s" % (job_template_fqname,
job_execution_id,
fabric_fq_name,
status,
completion_percent,
result, message))
except Exception as e:
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_LOG_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
fabric_name=fabric_fq_name,
exc_msg=repr(e))
raise JobException(msg, job_execution_id)
def send_job_execution_uve(
self,
fabric_fq_name,
job_template_fqname,
job_execution_id,
timestamp=None,
percentage_completed=None):
try:
fabric_job_name = list(job_template_fqname)
fabric_job_name.insert(0, fabric_fq_name)
fabric_job_uve_name = ':'.join(map(str, fabric_job_name))
job_exe_data = FabricJobExecution(
name=fabric_job_uve_name,
job_status='IN_PROGRESS',
percentage_completed=percentage_completed)
job_uve = FabricJobUve(
data=job_exe_data,
sandesh=self.config_logger._sandesh)
job_uve.send(sandesh=self.config_logger._sandesh)
except Exception as exp:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def send_prouter_job_uve(self,
job_template_fqname,
fq_names,
job_execution_id,
prouter_state=None,
job_status=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
if prouter_state is None:
prouter_job_data = PhysicalRouterJobExecution(
name=fq_names,
execution_id=job_execution_id,
job_status=job_status)
prouter_job_data = PhysicalRouterJobExecution(
name=fq_names,
execution_id=job_execution_id,
prouter_state=prouter_state,
job_status=job_status)
prouter_job_uve = PhysicalRouterJobUve(
data=prouter_job_data, sandesh=self.config_logger._sandesh)
prouter_job_uve.send(sandesh=self.config_logger._sandesh)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def test_execute_job_generic_exception(self):
# create job template
job_template = JobTemplate(job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=False,
job_template_playbooks=TestJobManagerUtils.
playbooks_list,
name='single_device_template_gen_exc')
job_template_uuid = self._vnc_lib.job_template_create(job_template)
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
job_template_uuid)
wm = WFManager(log_utils.get_config_logger(),
self._vnc_lib, job_input_json, log_utils)
# mock the job_handler to raise a general exception
mock_job_handler = flexmock()
flexmock(JobHandler).new_instances(mock_job_handler)
exc = Exception("Mock "
"Generic Exception")
flexmock(mock_job_handler).should_receive('handle_job') \
.and_raise(exc)
exc_msg = repr(exc)
sys_exit_msg = self.assertRaises(SystemExit, wm.start_job)
self.assertEqual(sys_exit_msg.code,
MsgBundle.getMessage(
MsgBundle.EXC_JOB_ERR_HDR) +
exc_msg + " ")
def send_job_log(self, job_template_fqname, job_execution_id,
fabric_fq_name, message, status, completion_percent=None,
result=None, timestamp=None, device_name=None,
details=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
if timestamp is None:
timestamp = int(round(time.time() * 1000))
details_str = json.dumps(details) if details else None
job_log_entry = JobLogEntry(
name=job_template_fqname, execution_id=job_execution_id,
fabric_name=fabric_fq_name, timestamp=timestamp,
message=message, status=status,
percentage_completed=completion_percent, result=result,
device_name=device_name, details=details_str)
job_log = JobLog(log_entry=job_log_entry)
job_log.send(sandesh=self.config_logger._sandesh)
self.config_logger.debug("Created job log for job template: %s, "
" execution id: %s, fabric_fq_name: %s"
"status: %s, completion_percent %s, "
"result: "
"%s, message: %s" % (job_template_fqname,
job_execution_id,
fabric_fq_name,
status,
completion_percent,
result, message))
except Exception as e:
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_LOG_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
fabric_name=fabric_fq_name,
exc_msg=repr(e))
raise JobException(msg, job_execution_id)
def test_execute_job_no_template_id(self):
job_input_json = {"no_template_id": "Missing template_id"}
exc_msg = self.assertRaises(Exception, JobManager, None, None,
job_input_json, None)
self.assertEqual(str(exc_msg),
MsgBundle.getMessage(MsgBundle.JOB_TEMPLATE_MISSING))
def initialize_sandesh_logger(self, config_args, sandesh=True,
sandesh_instance=None):
# parse the logger args
args = self.parse_logger_args(config_args)
args.random_collectors = args.collectors
if args.collectors:
args.random_collectors = random.sample(args.collectors,
len(args.collectors))
self.args = args
# initialize logger
logger = JobLogger(args=args,
sandesh_instance_id=self.sandesh_instance_id,
sandesh_instance=sandesh_instance)
if not sandesh_instance and sandesh:
try:
sandesh_util = SandeshUtils(logger)
sandesh_util.wait_for_connection_establish()
except JobException:
msg = MsgBundle.getMessage(
MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR)
raise JobException(msg)
logger.info("Sandesh is initialized."
" Config logger instance created.")
return logger
def send_job_execution_uve(self,
fabric_fq_name,
job_template_fqname,
job_execution_id,
timestamp=None,
percentage_completed=None):
try:
fabric_job_name = list(job_template_fqname)
fabric_job_name.insert(0, fabric_fq_name)
fabric_job_uve_name = ':'.join(map(str, fabric_job_name))
job_exe_data = FabricJobExecution(
name=fabric_job_uve_name,
job_status='IN_PROGRESS',
percentage_completed=percentage_completed)
job_uve = FabricJobUve(data=job_exe_data,
sandesh=self.config_logger._sandesh)
job_uve.send(sandesh=self.config_logger._sandesh)
except Exception as exp:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def handle_job(self, result_handler, job_percent_per_task,
device_id=None, device_name=None):
playbook_output = None
playbook_info = None
try:
msg = "Starting playbook execution for job template %s with " \
"execution id %s" % (self._job_template.get_uuid(),
self._execution_id)
self._logger.debug(msg)
# get the playbook information from the job template
playbook_info = self.get_playbook_info(job_percent_per_task,
device_id)
# run the playbook and retrieve the playbook output if any
playbook_output = self.run_playbook(
playbook_info,
result_handler.percentage_completed)
# retrieve the device_op_results in case it was set for
# generic device operations.
playbook_output_results = None
if playbook_output != None:
playbook_output_results = playbook_output.get('results')
msg = MsgBundle.getMessage(
MsgBundle.PLAYBOOK_EXECUTION_COMPLETE,
job_template_name=self._job_template.get_fq_name()[-1],
job_execution_id=self._execution_id)
self._logger.debug(msg)
result_handler.update_job_status(JobStatus.SUCCESS, msg,
device_id, device_name,
pb_results=playbook_output_results)
if playbook_output:
result_handler.update_playbook_output(playbook_output)
self.check_and_send_prouter_job_uve_for_multidevice(playbook_info,
JobStatus.SUCCESS.value,
playbook_output_results)
if self.current_percentage:
result_handler.percentage_completed = self.current_percentage
except JobException as job_exp:
self._logger.error("%s" % job_exp.msg)
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg,
device_id, device_name)
if playbook_info:
self.check_and_send_prouter_job_uve_for_multidevice(playbook_info,
JobStatus.FAILURE.value)
except Exception as exp:
self._logger.error("Error while executing job %s " % repr(exp))
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, exp.message,
device_id, device_name)
if playbook_info:
self.check_and_send_prouter_job_uve_for_multidevice(playbook_info,
JobStatus.FAILURE.value)
def test_execute_job_no_execution_id(self):
job_input_json = {"job_template_id": "random_template_id"}
exc_msg = self.assertRaises(Exception, JobManager, None, None,
job_input_json, None)
self.assertEqual(
str(exc_msg),
MsgBundle.getMessage(MsgBundle.JOB_EXECUTION_ID_MISSING))
def test_execute_job_input_schema(self):
# create job template
fake_job_template = flexmock(
job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=False,
job_template_playbooks=TestJobManagerUtils.
playbooks_list,
name='input_schema_template',
fq_name=["default-global-system-config",
"input_schema_template"],
uuid='random_uuid')
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
"random_uuid")
mock_vnc = flexmock()
flexmock(VncApi).new_instances(mock_vnc)
flexmock(mock_vnc).should_receive(
'job_template_read').with_args(
id="random_uuid").and_return(fake_job_template)
wm = WFManager(log_utils.get_config_logger(),
mock_vnc, job_input_json, log_utils)
exc = Exception("'name' is a required property")
fake_schema = TestJobManagerUtils.fake_schema
fake_job_template.should_receive(
'get_job_template_input_schema').and_return(fake_schema)
fake_job_template.should_receive('get_uuid').and_return(
'random_uuid')
# mock the job_handler to raise an exception
fake_job_template.should_receive('get_job_template_playbooks')
sys_exit_msg = self.assertRaises(SystemExit, wm.start_job)
self.assertEqual(sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_EXC_REC_HDR) +
MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA,
job_template_id="random_uuid",
exc_obj=exc) +
" ")
def test_execute_job_no_credentials(self):
# create job template
job_template = JobTemplate(job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=True,
job_template_playbooks=TestJobManagerUtils.
playbooks_list,
name='multi_device_no_credentials')
job_template_uuid = self._vnc_lib.job_template_create(job_template)
device_id = "aad74e24-a00b-4eb3-8412-f8b9412925c3"
TestJobManagerUtils.mock_sandesh_check()
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
job_template_uuid)
job_input_json.update({"params": {"device_list": [device_id]},
"device_json": {
device_id:
{"device_family": "MX",
"device_vendor": "Juniper",
"device_product":
"Some random product",
"device_fqname": [
"default-global-system-config",
"random_fqname"]}}})
wm = WFManager(log_utils.get_config_logger(),
self._vnc_lib, job_input_json, log_utils)
sys_exit_msg = self.assertRaises(SystemExit, wm.start_job)
self.assertEqual(
sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) +
MsgBundle.getMessage(MsgBundle.
JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) +
device_id +
",\n" +
MsgBundle.getMessage(MsgBundle.PLAYBOOK_RESULTS_MESSAGE) +
device_id +
":" +
MsgBundle.getMessage(
MsgBundle.NO_CREDENTIALS_FOUND,
device_id=device_id) +
" \n")
def _validate_job_input(self, input_schema, ip_json):
if ip_json is None:
msg = MsgBundle.getMessage(
MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND)
raise JobException(msg,
self.job_execution_id)
try:
ip_schema_json = input_schema
if isinstance(input_schema, str):
ip_schema_json = json.loads(input_schema)
jsonschema.validate(ip_json, ip_schema_json)
self._logger.debug("Input Schema Validation Successful"
"for template %s" % self.job_template_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA,
job_template_id=self.job_template_id,
exc_obj=exp)
raise JobException(msg, self.job_execution_id)
def close_sandesh_connection(self):
try:
self.wait_for_msg_send()
except JobException as job_exp:
msg = MsgBundle.getMessage(MsgBundle.CLOSE_SANDESH_EXCEPTION)
self._logger.error(msg)
job_exp.msg = msg
raise job_exp
finally:
self.uninit_sandesh()
def close_sandesh_connection(self):
try:
self.wait_for_msg_send()
except JobException as job_exp:
msg = MsgBundle.getMessage(MsgBundle.CLOSE_SANDESH_EXCEPTION)
self._logger.error(msg)
job_exp.msg = msg
raise job_exp
finally:
self.uninit_sandesh()
def test_execute_job_explicit_mismatch(self):
# create job template
job_template = JobTemplate(
job_template_type='device',
job_template_job_runtime='ansible',
job_template_multi_device_job=True,
job_template_playbooks=TestJobManagerUtils.playbooks_list,
name='multi_device_explicit_mismatch')
job_template_uuid = self._vnc_lib.job_template_create(job_template)
TestJobManagerUtils.mock_sandesh_check()
device_id = "aad74e24-a00b-4eb3-8412-f8b9412925c3"
job_input_json, log_utils = TestJobManagerUtils.get_min_details(
job_template_uuid)
job_input_json.update({
"params": {
"device_list": [device_id]
},
"device_json": {
device_id: {
"device_family": "MX",
"device_vendor": "Arista",
"device_username": "******",
"device_password": "******"
}
}
})
jmgr = JobManager(log_utils.get_config_logger(), self._vnc_lib,
job_input_json, log_utils)
sys_exit_msg = self.assertRaises(SystemExit, jmgr.start_job)
self.assertEqual(
sys_exit_msg.code,
MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) +
MsgBundle.getMessage(
MsgBundle.JOB_MULTI_DEVICE_FAILED_MESSAGE_HDR) + device_id +
",\n" + MsgBundle.getMessage(MsgBundle.PLAYBOOK_RESULTS_MESSAGE) +
device_id + ":" +
MsgBundle.getMessage(MsgBundle.PLAYBOOK_INFO_DEVICE_MISMATCH,
device_vendor="Arista",
device_family="MX") + " \n")
def run_playbook(self, playbook_info, percentage_completed):
playbook_output = None
try:
# create job log to capture the start of the playbook
device_name = \
playbook_info['extra_vars']['playbook_input'].get(
'device_fqname')
if device_name:
device_name = device_name[-1]
playbook_name = playbook_info['uri'].split('/')[-1]
msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
if not os.path.exists(playbook_info['uri']):
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND,
playbook_uri=playbook_info['uri'])
raise JobException(msg,
self._execution_id)
# Run playbook in a separate process. This is needed since
# ansible cannot be used in a greenlet based patched environment
playbook_output = self.run_playbook_process(playbook_info,
percentage_completed)
# create job log to capture completion of the playbook execution
msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
return playbook_output
except JobException:
raise
except Exception as exp:
trace = traceback.format_exc()
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException("%s\n%s" %(msg, trace), self._execution_id)
def send_prouter_object_log(self, prouter_fqname, job_execution_id,
job_input, job_template_fqname,
onboarding_state,
os_version=None, serial_num=None,
timestamp=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
prouter_fqname = self.get_fq_name_log_str(prouter_fqname)
if timestamp is None:
timestamp = int(round(time.time() * 1000))
# create the prouter object log
prouter_log_entry = PRouterOnboardingLogEntry(
name=prouter_fqname,
job_execution_id=job_execution_id,
os_version=os_version,
serial_num=serial_num,
onboarding_state=onboarding_state,
timestamp=timestamp,
job_template_fqname=job_template_fqname,
job_input=job_input)
prouter_log = PRouterOnboardingLog(log_entry=prouter_log_entry)
prouter_log.send(sandesh=self.config_logger._sandesh)
self.config_logger.debug(
"Created prouter object log for router: %s, "
" execution id: %s, job_template: %s, os_version: "
"%s, serial_num: %s, onboarding_state %s" %
(prouter_fqname,
job_execution_id,
job_template_fqname,
os_version,
serial_num,
onboarding_state))
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.SEND_PROUTER_OBJECT_LOG_ERROR,
prouter_fqname=prouter_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def start_job(self):
# spawn job greenlets
job_handler = JobHandler(self._logger, self._vnc_api,
self.job_template, self.job_execution_id,
self.job_data, self.job_utils,
self.device_json, self.auth_token,
self.api_server_host, self.job_log_utils,
self.sandesh_args, self.fabric_fq_name,
self.job_log_utils.args.playbook_timeout,
self.playbook_seq, self.vnc_api_init_params,
self._zk_client, self.db_init_params,
self.cluster_id)
if self.device_json is not None:
if not self.device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg, self.job_execution_id)
else:
self.handle_multi_device_job(job_handler, self.result_handler)
else:
self.handle_single_job(job_handler, self.result_handler)
def send_prouter_job_uve(
self,
job_template_fqname,
fq_names,
job_execution_id,
prouter_state=None,
job_status=None,
percentage_completed=None,
device_op_results = "{}"):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
if prouter_state is None:
prouter_job_data = PhysicalRouterJobExecution(
name=fq_names,
execution_id=job_execution_id,
job_status=job_status,
percentage_completed=percentage_completed,
device_op_results = device_op_results
)
else:
prouter_job_data = PhysicalRouterJobExecution(
name=fq_names,
execution_id=job_execution_id,
prouter_state=prouter_state,
job_status=job_status,
percentage_completed=percentage_completed,
device_op_results = device_op_results
)
prouter_job_uve = PhysicalRouterJobUve(
data=prouter_job_data, sandesh=self.config_logger._sandesh)
prouter_job_uve.send(sandesh=self.config_logger._sandesh)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def initialize_sandesh_logger(self, config_args, sandesh=True,
sandesh_instance=None):
# parse the logger args
args = self.parse_logger_args(config_args)
args.random_collectors = args.collectors
if args.collectors:
args.random_collectors = random.sample(args.collectors,
len(args.collectors))
self.args = args
# initialize logger
logger = JobLogger(args=args,
sandesh_instance_id=self.sandesh_instance_id,
sandesh_instance=sandesh_instance)
if not sandesh_instance and sandesh:
try:
sandesh_util = SandeshUtils(logger)
sandesh_util.wait_for_connection_establish()
except JobException:
msg = MsgBundle.getMessage(
MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR)
raise JobException(msg)
logger.info("Sandesh is initialized. Config logger instance created.")
return logger
def handle_job(self, result_handler, job_percent_per_task,
device_id=None, device_name=None):
playbook_output = None
playbook_info = None
try:
msg = "Starting playbook execution for job template %s with " \
"execution id %s" % (self._job_template.get_uuid(),
self._execution_id)
self._logger.debug(msg)
# Always acquire the lock while executing the multi device jobs
if device_id is not None:
if not self._acquire_device_lock(device_id):
raise JobException(MsgBundle.getMessage(
MsgBundle.DEVICE_LOCK_FAILURE))
# get the playbook information from the job template
playbook_info = self.get_playbook_info(job_percent_per_task,
device_id)
# run the playbook and retrieve the playbook output if any
playbook_output = self.run_playbook(
playbook_info,
result_handler.percentage_completed)
# retrieve the device_op_results in case it was set for
# generic device operations.
playbook_output_results = None
if playbook_output != None:
playbook_output_results = playbook_output.get('results')
msg = MsgBundle.getMessage(
MsgBundle.PLAYBOOK_EXECUTION_COMPLETE,
job_template_name=self._job_template.get_fq_name()[-1],
job_execution_id=self._execution_id)
self._logger.debug(msg)
result_handler.update_job_status(
JobStatus.SUCCESS, msg, device_id, device_name,
pb_results=playbook_output_results)
if playbook_output:
result_handler.update_playbook_output(playbook_output)
self.check_and_send_prouter_job_uve_for_multidevice(
playbook_info, JobStatus.SUCCESS.value,
playbook_output_results)
if self.current_percentage:
result_handler.percentage_completed = self.current_percentage
except JobException as job_exp:
self._logger.error("%s" % job_exp.msg)
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg,
device_id, device_name)
if playbook_info:
self.check_and_send_prouter_job_uve_for_multidevice(
playbook_info, JobStatus.FAILURE.value)
except Exception as exp:
self._logger.error("Error while executing job %s " % repr(exp))
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, exp.message,
device_id, device_name)
if playbook_info:
self.check_and_send_prouter_job_uve_for_multidevice(
playbook_info, JobStatus.FAILURE.value)
finally:
if device_id is not None:
self._release_device_lock(device_id)
def get_playbook_info(self, job_percent_per_task, device_id=None):
try:
# create the cmd line param for the playbook
extra_vars = {
'input': self._job_input,
'job_template_id': self._job_template.get_uuid(),
'job_template_fqname': self._job_template.fq_name,
'fabric_fq_name': self._fabric_fq_name,
'auth_token': self._auth_token,
'api_server_host': self._api_server_host,
'job_execution_id': self._execution_id,
'args': self._sandesh_args,
'vnc_api_init_params': self._vnc_api_init_params,
'db_init_params': self._db_init_params,
'cluster_id': self._cluster_id,
'playbook_job_percentage': job_percent_per_task
}
playbooks = self._job_template.get_job_template_playbooks()
if device_id:
if not self._device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg,
self._execution_id)
device_data = self._device_json.get(device_id)
if not device_data:
msg = MsgBundle.getMessage(MsgBundle.NO_DEVICE_DATA_FOUND,
device_id=device_id)
raise JobException(msg, self._execution_id)
device_family = device_data.get('device_family')
device_vendor = device_data.get('device_vendor')
device_product = device_data.get('device_product')
if not device_vendor or not device_family:
msg = MsgBundle.getMessage(MsgBundle.
DEVICE_VENDOR_FAMILY_MISSING,
device_id=device_id)
raise JobException(msg, self._execution_id)
if not device_product:
msg = MsgBundle.getMessage(MsgBundle.
PRODUCT_NAME_MISSING,
device_id=device_id)
raise JobException(msg, self._execution_id)
# check for credentials,required param; else playbooks
# will fail
device_username = device_data.get('device_username')
device_password = device_data.get('device_password')
if not device_username or not device_password:
msg = MsgBundle.getMessage(MsgBundle.
NO_CREDENTIALS_FOUND,
device_id=device_id)
raise JobException(msg, self._execution_id)
# update extra-vars to reflect device-related params
device_fqname = device_data.get('device_fqname')
device_management_ip = device_data.get('device_management_ip')
image_uuid = device_data.get('device_image_uuid')
hitless_upgrade = device_data.get('device_hitless_upgrade')
extra_vars.update({
'device_id': device_id,
'device_fqname': device_fqname,
'device_management_ip':
device_management_ip,
'vendor': device_vendor,
'device_family': device_family,
'device_username': device_username,
'device_password': device_password,
'product_name': device_product,
'device_image_uuid': image_uuid,
'device_hitless_upgrade': hitless_upgrade
})
self._logger.debug("Passing the following device "
"ip to playbook %s " % device_management_ip)
# get the playbook uri from the job template
play_info = playbooks.playbook_info[self._playbook_seq]
playbook_input = {'playbook_input': extra_vars}
playbook_info = dict()
playbook_info['uri'] = play_info.playbook_uri
playbook_info['extra_vars'] = playbook_input
return playbook_info
except JobException:
raise
except Exception as exp:
msg = MsgBundle.getMessage(
MsgBundle.GET_PLAYBOOK_INFO_ERROR,
job_template_id=self._job_template.get_uuid(),
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
def parse_args():
parser = argparse.ArgumentParser(description='Ansible playbook input '
'parameters')
parser.add_argument('-i', '--playbook_input', nargs=1,
help='Playbook input json')
return parser.parse_args()
if __name__ == "__main__":
playbook_input_json = None
try:
playbook_params = parse_args()
playbook_input_json = json.loads(playbook_params.playbook_input[0])
if playbook_input_json is None:
sys.exit(MsgBundle.getMessage(MsgBundle.NO_PLAYBOOK_INPUT_DATA))
except Exception as exp:
ERR_MSG = "Failed to start playbook due "\
"to Exception: %s" % traceback.print_stack()
JM_LOGGER.error(ERR_MSG)
sys.exit(MsgBundle.getMessage(MsgBundle.PLAYBOOK_INPUT_PARSING_ERROR,
exc_msg=repr(exp)))
playbook_helper = PlaybookHelper()
pb_output = playbook_helper.execute_playbook(playbook_input_json)
# if it comes here, it implies the pb_output is of correct
# format and is present with status Sucess. So derive the
# playbook output to be written to file and finally write END to the file
unique_pb_id = playbook_input_json['extra_vars'][
'playbook_input']['unique_pb_id']
def execute_playbook(self, playbook_info):
output = None
try:
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources=['localhost'])
variable_manager = VariableManager(loader=loader,
inventory=inventory)
Options = namedtuple('Options',
['listtags', 'listtasks', 'listhosts',
'syntax', 'connection', 'module_path',
'forks', 'remote_user', 'private_key_file',
'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args',
'become', 'become_method', 'become_user',
'verbosity', 'check', 'diff'])
options = Options(listtags=False, listtasks=False, listhosts=False,
syntax=False, connection='ssh', module_path=None,
forks=100, remote_user=None,
private_key_file=None, ssh_common_args=None,
ssh_extra_args=None, sftp_extra_args=None,
scp_extra_args=None, become=None,
become_method=None, become_user=None,
verbosity=None, check=False, diff=False)
variable_manager.extra_vars = playbook_info['extra_vars']
pbex = PlaybookExecutor(playbooks=[playbook_info['uri']],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options, passwords=None)
ret_val = pbex.run()
output = self.get_plugin_output(pbex)
if ret_val != 0:
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_RETURN_WITH_ERROR)
raise Exception(msg)
if output is None or output.get('status') is None:
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_OUTPUT_MISSING)
raise Exception(msg)
if output.get('status').lower() == "failure":
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_STATUS_FAILED)
raise Exception(msg)
return output
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXECUTE_ERROR,
playbook_uri=playbook_info['uri'],
execution_id=playbook_info['extra_vars']
['playbook_input']['job_execution_id'],
exc_msg=repr(exp))
if exp.message:
msg = msg + "\n" + exp.message
JM_LOGGER.error(msg)
# after handling exception, write an END
# to stop listening to the file if created
unique_pb_id = playbook_info['extra_vars'][
'playbook_input']['unique_pb_id']
exec_id = playbook_info['extra_vars']['playbook_input'][
'job_execution_id']
self._job_file_write.write_to_file(
exec_id,
unique_pb_id,
JobFileWrite.PLAYBOOK_OUTPUT,
json.dumps(output)
)
with open("/tmp/"+exec_id, "a") as f:
f.write(unique_pb_id + 'END' + PLAYBOOK_EOL_PATTERN)
sys.exit(msg)
def start_job(self):
job_error_msg = None
job_template = None
try:
# create job UVE and log
self.result_handler = JobResultHandler(self.job_template_id,
self.job_execution_id,
self.fabric_fq_name,
self._logger,
self.job_utils,
self.job_log_utils)
job_template = self.job_utils.read_job_template()
msg = MsgBundle.getMessage(MsgBundle.START_JOB_MESSAGE,
job_execution_id=self.job_execution_id,
job_template_name=\
job_template.fq_name[-1])
self._logger.debug(msg)
timestamp = int(round(time.time() * 1000))
self.job_log_utils.send_job_log(job_template.fq_name,
self.job_execution_id,
self.fabric_fq_name,
msg,
JobStatus.STARTING.value,
timestamp=timestamp)
# validate job input if required by job_template input_schema
input_schema = job_template.get_job_template_input_schema()
if input_schema:
self._validate_job_input(input_schema, self.job_data)
playbook_list = job_template.get_job_template_playbooks()\
.get_playbook_info()
job_percent = None
# calculate job percentage for each playbook
if len(playbook_list) > 1:
task_weightage_array = [
pb_info.job_completion_weightage
for pb_info in playbook_list]
for i in range(0, len(playbook_list)):
if len(playbook_list) > 1:
# get the job percentage based on weightage of each plabook
# when they are chained
job_percent = \
self.job_log_utils.calculate_job_percentage(
len(playbook_list), buffer_task_percent=True,
total_percent=100, task_seq_number=i + 1,
task_weightage_array=task_weightage_array)[0]
else:
job_percent = \
self.job_log_utils.calculate_job_percentage(
len(playbook_list), buffer_task_percent=True,
total_percent=100)[0] # using equal weightage
retry_devices = None
while True:
job_mgr = JobManager(self._logger, self._vnc_api,
self.job_input, self.job_log_utils,
job_template,
self.result_handler, self.job_utils, i,
job_percent, self._zk_client,
self.db_init_params,
self.cluster_id)
job_mgr.start_job()
# retry the playbook execution if retry_devices is added to
# the playbook output
job_status = self.result_handler.job_result_status
retry_devices = self.result_handler.get_retry_devices()
if job_status == JobStatus.FAILURE or not retry_devices:
break
self.job_input['device_json'] = retry_devices
# stop the workflow if playbook failed
if self.result_handler.job_result_status == JobStatus.FAILURE:
# stop workflow only if its a single device job or
# it is a multi device playbook
# and all the devices have failed some job execution
# declare it as failure and the stop the workflow
if self.job_input.get('device_json') is None or\
len(self.result_handler.failed_device_jobs)\
== len(self.job_input.get('device_json')):
self._logger.error(
"Stop the workflow on the failed Playbook.")
break
elif not retry_devices:
# it is a multi device playbook but one of the device jobs
# have failed. This means we should still declare
# the operation as success. We declare workflow as
# success even if one of the devices has succeeded the job
self.result_handler.job_result_status = JobStatus.SUCCESS
# update the job input with marked playbook output json
pb_output = self.result_handler.playbook_output or {}
# read the device_data output of the playbook
# and update the job input so that it can be used in next
# iteration
if not self.job_input.get('device_json'):
device_json = pb_output.pop('device_json', None)
self.job_input['device_json'] = device_json
self.job_input.get('input', {}).update(pb_output)
# create job completion log and update job UVE
self.result_handler.create_job_summary_log(
job_template.fq_name)
# in case of failures, exit the job manager process with failure
if self.result_handler.job_result_status == JobStatus.FAILURE:
job_error_msg = self.result_handler.job_summary_message
except JobException as exp:
err_msg = "Job Exception recieved: %s " % repr(exp)
self._logger.error(err_msg)
self._logger.error("%s" % traceback.format_exc())
self.result_handler.update_job_status(JobStatus.FAILURE,
err_msg)
if job_template:
self.result_handler.create_job_summary_log(
job_template.fq_name)
job_error_msg = err_msg
except Exception as exp:
err_msg = "Error while executing job %s " % repr(exp)
self._logger.error(err_msg)
self._logger.error("%s" % traceback.format_exc())
self.result_handler.update_job_status(JobStatus.FAILURE,
err_msg)
self.result_handler.create_job_summary_log(job_template.fq_name)
job_error_msg = err_msg
finally:
# need to wait for the last job log and uve update to complete
# via sandesh and then close sandesh connection
sandesh_util = SandeshUtils(self._logger)
sandesh_util.close_sandesh_connection()
self._logger.info("Closed Sandesh connection")
if job_error_msg is not None:
sys.exit(job_error_msg)
