def run_playbook_process(self, playbook_info, percentage_completed):
playbook_process = None
self.current_percentage = percentage_completed
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
unique_pb_id = str(uuid.uuid4())
playbook_info['extra_vars']['playbook_input']['unique_pb_id']\
= unique_pb_id
exec_id =\
playbook_info['extra_vars']['playbook_input'][
'job_execution_id']
pr_object_log_start_time = time.time()
playbook_process = subprocess32.Popen(["python",
playbook_exec_path,
"-i",
json.dumps(playbook_info)],
close_fds=True, cwd='/')
# this is to yield the context to the playbooks so that
# they start running concurrently
time.sleep(0.5)
marked_output = self.process_file_and_get_marked_output(
unique_pb_id, exec_id, playbook_process
)
marked_jsons = self._extract_marked_json(marked_output)
self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
pr_object_log_end_time = time.time()
self.send_pr_object_log(
exec_id,
pr_object_log_start_time,
pr_object_log_end_time,
playbook_process.returncode)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(
MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.
RUN_PLAYBOOK_PROCESS_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
raise JobException(msg, self._execution_id)
def read_fabric_data(self,
request_params,
job_execution_id,
is_delete=False):
if request_params.get('input') is None:
err_msg = "Missing job input"
raise JobException(err_msg, job_execution_id)
fabric_fq_name = None
if request_params.get('input').get('fabric_fq_name'):
fabric_fq_name = request_params.get('input').get('fabric_fq_name')
elif request_params.get('input').get('fabric_uuid'):
# get the fabric fq_name from the db if fabric_uuid is provided
fabric_uuid = request_params.get('input').get('fabric_uuid')
try:
fabric_fq_name = self._db_conn.uuid_to_fq_name(fabric_uuid)
except NoIdError as e:
raise JobException(str(e), job_execution_id)
else:
if "device_deletion_template" in request_params.get(
'job_template_fq_name'):
fabric_fq_name = ["__DEFAULT__"]
elif not is_delete:
err_msg = "Missing fabric details in the job input"
raise JobException(err_msg, job_execution_id)
if fabric_fq_name:
fabric_fq_name_str = ':'.join(map(str, fabric_fq_name))
request_params['fabric_fq_name'] = fabric_fq_name_str
def run_playbook_process(self, playbook_info, percentage_completed):
playbook_process = None
self.current_percentage = percentage_completed
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
unique_pb_id = str(uuid.uuid4())
playbook_info['extra_vars']['playbook_input']['unique_pb_id']\
= unique_pb_id
exec_id =\
playbook_info['extra_vars']['playbook_input'][
'job_execution_id']
pr_uve_name = self.get_pr_uve_name_from_device_name(playbook_info)
playbook_process = subprocess32.Popen(["python",
playbook_exec_path,
"-i",
json.dumps(playbook_info)],
close_fds=True, cwd='/')
# Save process ID in case of abort
self._pb_pids.append(playbook_process.pid)
# this is to yield the context to the playbooks so that
# they start running concurrently
gevent.sleep(0)
marked_output = self.process_file_and_get_marked_output(
unique_pb_id, exec_id, playbook_process, pr_uve_name)
marked_jsons = self._extract_marked_json(marked_output)
playbook_output = marked_jsons.get(JobFileWrite.PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
# create prouter UVE in job_manager only if it is not a multi
# device job template
if not self.is_multi_device_playbook:
status = "SUCCESS" if playbook_process.returncode == 0 \
else "FAILURE"
self.send_prouter_uve(exec_id, status)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(
MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.
PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
if playbook_output:
msg = "%s\n Error Message from playbook: %s" % (
msg, playbook_output.get('message', "")
)
raise JobException(msg, self._execution_id)
return playbook_output
def calculate_job_percentage(self, num_tasks, buffer_task_percent=False,
task_seq_number=None, total_percent=100,
task_weightage_array=None):
if num_tasks is None:
raise JobException("Number of tasks is required to calculate "
"the job percentage")
try:
getcontext().prec = 3
# Use buffered approach to mitigate the cumulative task percentages
# exceeding the total job percentage
if buffer_task_percent:
buffer_percent = 0.05 * total_percent
total_percent -= buffer_percent
# if task weightage is not provided, the task will recive an
# equally divided chunk from the total_percent based on the total
# number of tasks
if task_weightage_array is None:
success_task_percent = float(old_div(Decimal(total_percent),
Decimal(num_tasks)))
else:
if task_seq_number is None:
raise JobException("Unable to calculate the task "
"percentage since the task sequence "
"number is not provided")
success_task_percent = float(old_div(Decimal(
task_weightage_array[task_seq_number - 1] *
total_percent), 100))
# based on the task sequence number calculate the percentage to be
# marked in cases of error. This is required to mark the job to
# 100% completion in cases of errors when successor tasks will not
# be executed.
failed_task_percent = None
if task_seq_number:
if task_weightage_array is None:
failed_task_percent = (num_tasks - task_seq_number + 1) *\
success_task_percent
else:
failed_task_percent = 0.00
for task_index in range(task_seq_number, num_tasks):
task_percent = float(old_div(Decimal(
task_weightage_array[task_index - 1] *
total_percent), 100))
failed_task_percent += task_percent
self.config_logger.info("success_task_percent %s "
"failed_task_percent %s " %
(success_task_percent,
failed_task_percent))
return success_task_percent, failed_task_percent
except Exception as e:
msg = "Exception while calculating the job pecentage %s " % repr(e)
self.config_logger.error(msg)
self.config_logger.error("%s" % traceback.format_exc())
raise JobException(e)
def read_device_data(self, device_list, request_params, job_exec_id):
device_data = dict()
for device_id in device_list:
try:
(ok, result) = self.db_read("physical-router", device_id, [
'physical_router_user_credentials',
'physical_router_management_ip', 'fq_name',
'physical_router_device_family',
'physical_router_vendor_name',
'physical_router_product_name', 'fabric_refs'
])
if not ok:
msg = "Error while reading the physical router " \
"with id %s : %s" % (device_id, result)
raise JobException(msg, job_exec_id)
except Exception as e:
msg = "Exception while reading device %s %s " % \
(device_id, str(e))
raise JobException(msg, job_exec_id)
device_json = {
"device_management_ip":
result.get('physical_router_management_ip')
}
device_json.update({"device_fqname": result.get('fq_name')})
user_cred = result.get('physical_router_user_credentials')
if user_cred:
device_json.update(
{"device_username": user_cred.get('username')})
device_json.update(
{"device_password": user_cred.get('password')})
device_family = result.get("physical_router_device_family")
if device_family:
device_json.update({"device_family": device_family})
device_vendor_name = result.get("physical_router_vendor_name")
if device_vendor_name:
device_json.update({"device_vendor": device_vendor_name})
device_product_name = result.get("physical_router_product_name")
if device_product_name:
device_json.update({"device_product": device_product_name})
device_data.update({device_id: device_json})
fabric_refs = result.get('fabric_refs')
if fabric_refs and len(fabric_refs) > 0:
fabric_fq_name = result.get('fabric_refs')[0].get('to')
fabric_fq_name_str = ':'.join(map(str, fabric_fq_name))
request_params['fabric_fq_name'] = fabric_fq_name_str
if len(device_data) > 0:
request_params.update({"device_json": device_data})
def run_playbook_process(self, playbook_info):
playbook_process = None
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
playbook_process = subprocess32.Popen([
"python", playbook_exec_path, "-i",
json.dumps(playbook_info)
],
close_fds=True,
cwd='/',
stdout=subprocess32.PIPE)
marked_output = {}
markers = [DEVICE_DATA, PLAYBOOK_OUTPUT]
while True:
output = playbook_process.stdout.readline()
if output == '' and playbook_process.poll() is not None:
break
if output:
self._logger.debug(output)
# read device list data and store in variable result handler
for marker in markers:
if marker in output:
marked_output[marker] = output
marked_jsons = self._extract_marked_json(marked_output)
self._device_data = marked_jsons.get(DEVICE_DATA)
self._playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed):
playbook_output = None
try:
# create job log to capture the start of the playbook
device_name = \
playbook_info['extra_vars']['playbook_input'].get(
'device_fqname')
if device_name:
device_name = device_name[-1]
playbook_name = playbook_info['uri'].split('/')[-1]
msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg,
JobStatus.IN_PROGRESS.value,
device_name=device_name)
if not os.path.exists(playbook_info['uri']):
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND,
playbook_uri=playbook_info['uri'])
raise JobException(msg, self._execution_id)
# Run playbook in a separate process. This is needed since
# ansible cannot be used in a greenlet based patched environment
playbook_output = self.run_playbook_process(
playbook_info, percentage_completed)
# create job log to capture completion of the playbook execution
msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg,
JobStatus.IN_PROGRESS.value,
device_name=device_name)
return playbook_output
except JobException:
raise
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info, percentage_completed):
try:
# create job log to capture the start of the playbook
device_id = \
playbook_info['extra_vars']['playbook_input'].get(
'device_id', "")
msg = MsgBundle.getMessage(MsgBundle.START_EXECUTE_PLAYBOOK_MSG,
playbook_uri=playbook_info['uri'],
device_id=device_id,
input_params=json.dumps(
playbook_info['extra_vars']
['playbook_input']
['input']),
extra_params=json.dumps(
playbook_info['extra_vars']
['playbook_input']
['params']))
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg, JobStatus.IN_PROGRESS.value)
if not os.path.exists(playbook_info['uri']):
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND,
playbook_uri=playbook_info['uri'])
raise JobException(msg,
self._execution_id)
# Run playbook in a separate process. This is needed since
# ansible cannot be used in a greenlet based patched environment
self.run_playbook_process(playbook_info, percentage_completed)
# create job log to capture completion of the playbook execution
msg = MsgBundle.getMessage(MsgBundle.PB_EXEC_COMPLETE_WITH_INFO,
playbook_uri=playbook_info['uri'],
device_id=device_id)
self._logger.debug(msg)
self._job_log_utils.send_job_log(self._job_template.fq_name,
self._execution_id,
self._fabric_fq_name,
msg, JobStatus.IN_PROGRESS.value)
except JobException:
raise
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
def send_job_log(self, job_template_fqname, job_execution_id,
fabric_fq_name, message, status, completion_percent=None,
result=None, timestamp=None, device_name=None,
details=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
if timestamp is None:
timestamp = int(round(time.time() * 1000))
details_str = json.dumps(details) if details else None
job_log_entry = JobLogEntry(
name=job_template_fqname, execution_id=job_execution_id,
fabric_name=fabric_fq_name, timestamp=timestamp,
message=message, status=status,
percentage_completed=completion_percent, result=result,
device_name=device_name, details=details_str)
job_log = JobLog(log_entry=job_log_entry)
job_log.send(sandesh=self.config_logger._sandesh)
self.config_logger.debug("Created job log for job template: %s, "
" execution id: %s, fabric_fq_name: %s"
"status: %s, completion_percent %s, "
"result: "
"%s, message: %s" % (job_template_fqname,
job_execution_id,
fabric_fq_name,
status,
completion_percent,
result, message))
except Exception as e:
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_LOG_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
fabric_name=fabric_fq_name,
exc_msg=repr(e))
raise JobException(msg, job_execution_id)
def _validate_job_input(self, input_schema, ip_json):
if ip_json is None:
msg = MsgBundle.getMessage(MsgBundle.INPUT_SCHEMA_INPUT_NOT_FOUND)
raise JobException(msg, self.job_execution_id)
try:
ip_schema_json = input_schema
if isinstance(input_schema, str):
ip_schema_json = json.loads(input_schema)
jsonschema.validate(ip_json, ip_schema_json)
self._logger.debug("Input Schema Validation Successful"
"for template %s" % self.job_template_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.INVALID_SCHEMA,
job_template_id=self.job_template_id,
exc_obj=exp)
raise JobException(msg, self.job_execution_id)
def send_prouter_job_uve(self,
job_template_fqname,
fq_names,
job_execution_id,
prouter_state=None,
job_status=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
if prouter_state is None:
prouter_job_data = PhysicalRouterJobExecution(
name=fq_names,
execution_id=job_execution_id,
job_status=job_status)
prouter_job_data = PhysicalRouterJobExecution(
name=fq_names,
execution_id=job_execution_id,
prouter_state=prouter_state,
job_status=job_status)
prouter_job_uve = PhysicalRouterJobUve(
data=prouter_job_data, sandesh=self.config_logger._sandesh)
prouter_job_uve.send(sandesh=self.config_logger._sandesh)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def initialize_sandesh_logger(self, config_args, sandesh=True,
sandesh_instance=None):
# parse the logger args
args = self.parse_logger_args(config_args)
args.random_collectors = args.collectors
if args.collectors:
args.random_collectors = random.sample(args.collectors,
len(args.collectors))
self.args = args
# initialize logger
logger = JobLogger(args=args,
sandesh_instance_id=self.sandesh_instance_id,
sandesh_instance=sandesh_instance)
if not sandesh_instance and sandesh:
try:
sandesh_util = SandeshUtils(logger)
sandesh_util.wait_for_connection_establish()
except JobException:
msg = MsgBundle.getMessage(
MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR)
raise JobException(msg)
logger.info("Sandesh is initialized."
" Config logger instance created.")
return logger
def start_job(self):
# spawn job greenlets
job_handler = JobHandler(self._logger, self._vnc_api,
self.job_template, self.job_execution_id,
self.job_data, self.job_utils,
self.device_json, self.auth_token,
self.contrail_cluster_id,
self.api_server_host, self.job_log_utils,
self.sandesh_args, self.fabric_fq_name,
self.job_log_utils.args.playbook_timeout,
self.playbook_seq, self.vnc_api_init_params,
self._zk_client)
self.job_handler = job_handler
# check if its a multi device playbook
playbooks = self.job_template.get_job_template_playbooks()
play_info = playbooks.playbook_info[self.playbook_seq]
is_multi_device_playbook = play_info.multi_device_playbook
# for fabric config push as part of delete workflow,
# device json is not needed. There will be no performance
# impact as fabric delete from DM will always have one prouter
# uuid in the device_list.
if is_multi_device_playbook:
if self.device_json is None or not self.device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg, self.job_execution_id)
else:
self.handle_multi_device_job(job_handler, self.result_handler)
else:
self.handle_single_job(job_handler, self.result_handler)
def send_job_execution_uve(self,
fabric_fq_name,
job_template_fqname,
job_execution_id,
timestamp=None,
percentage_completed=None):
try:
fabric_job_name = list(job_template_fqname)
fabric_job_name.insert(0, fabric_fq_name)
fabric_job_uve_name = ':'.join(map(str, fabric_job_name))
job_exe_data = FabricJobExecution(
name=fabric_job_uve_name,
job_status='IN_PROGRESS',
percentage_completed=percentage_completed)
job_uve = FabricJobUve(data=job_exe_data,
sandesh=self.config_logger._sandesh)
job_uve.send(sandesh=self.config_logger._sandesh)
except Exception as exp:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def get_job_concurrency(self, job_template_id, job_exec_id):
(ok, result) = self.db_read("job-template", job_template_id,
['job_template_concurrency_level'])
if not ok:
msg = "Error while reading the job concurrency " \
"from the job template with id %s : %s" %\
(job_template_id, result)
raise JobException(msg, job_exec_id)
return result.get('job_template_concurrency_level')
def test_sandesh_timeout(self):
mocked_sandesh_utils = flexmock()
flexmock(SandeshUtils, __new__=mocked_sandesh_utils)
mocked_sandesh_utils.should_receive('__init__')
mocked_sandesh_utils.should_receive('wait_for_connection_establish')\
.and_raise(JobException())
args = {"collectors": ['127.0.0.1:8086']}
exc_msg = self.assertRaises(JobException, JobLogUtils, "rdm_exc_id",
json.dumps(args))
self.assertEqual(
str(exc_msg),
"JobException in execution" + " (None): " + MsgBundle.getMessage(
MsgBundle.SANDESH_INITIALIZATION_TIMEOUT_ERROR))
def run_playbook(self, playbook_info, percentage_completed):
try:
# create job log to capture the start of the playbook
device_name = \
playbook_info['extra_vars']['playbook_input'].get(
'device_fqname')
if device_name:
device_name = device_name[-1]
playbook_name = playbook_info['uri'].split('/')[-1]
msg = MsgBundle.getMessage(MsgBundle.START_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
if not os.path.exists(playbook_info['uri']):
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_NOT_FOUND,
playbook_uri=playbook_info['uri'])
raise JobException(msg,
self._execution_id)
# Run playbook in a separate process. This is needed since
# ansible cannot be used in a greenlet based patched environment
playbook_output = self.run_playbook_process(playbook_info,
percentage_completed)
# create job log to capture completion of the playbook execution
msg = MsgBundle.getMessage(MsgBundle.STOP_EXE_PB_MSG,
playbook_name=playbook_name)
self._logger.debug(msg)
return playbook_output
except JobException:
raise
except Exception as exp:
trace = traceback.format_exc()
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException("%s\n%s" % (msg, trace), self._execution_id)
def start_job(self):
# spawn job greenlets
job_handler = JobHandler(
self._logger, self._vnc_api, self.job_template,
self.job_execution_id, self.job_data, self.job_utils,
self.device_json, self.auth_token, self.api_server_host,
self.job_log_utils, self.sandesh_args, self.fabric_fq_name,
self.job_log_utils.args.playbook_timeout, self.playbook_seq)
if self.device_json is not None:
if not self.device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg, self.job_execution_id)
else:
self.handle_multi_device_job(job_handler, self.result_handler)
else:
self.handle_single_job(job_handler, self.result_handler)
def send_prouter_object_log(self,
prouter_fqname,
job_execution_id,
job_input,
job_template_fqname,
onboarding_state,
os_version=None,
serial_num=None,
timestamp=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
prouter_fqname = self.get_fq_name_log_str(prouter_fqname)
if timestamp is None:
timestamp = int(round(time.time() * 1000))
# create the prouter object log
prouter_log_entry = PRouterOnboardingLogEntry(
name=prouter_fqname,
job_execution_id=job_execution_id,
os_version=os_version,
serial_num=serial_num,
onboarding_state=onboarding_state,
timestamp=timestamp,
job_template_fqname=job_template_fqname,
job_input=job_input)
prouter_log = PRouterOnboardingLog(log_entry=prouter_log_entry)
prouter_log.send(sandesh=self.config_logger._sandesh)
self.config_logger.debug(
"Created prouter object log for router: %s, "
" execution id: %s, job_template: %s, os_version: "
"%s, serial_num: %s, onboarding_state %s" %
(prouter_fqname, job_execution_id, job_template_fqname,
os_version, serial_num, onboarding_state))
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.SEND_PROUTER_OBJECT_LOG_ERROR,
prouter_fqname=prouter_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
def send_job_execution_uve(self,
job_template_fqname,
job_execution_id,
timestamp=None,
percentage_completed=None):
try:
job_template_fqname = self.get_fq_name_log_str(job_template_fqname)
if timestamp is None:
timestamp = int(round(time.time() * 1000))
job_exe_data = JobExecution(
name=job_template_fqname,
execution_id=job_execution_id,
job_start_ts=timestamp,
percentage_completed=percentage_completed)
job_uve = UveJobExecution(data=job_exe_data)
job_uve.send(sandesh=self.config_logger._sandesh)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.SEND_JOB_EXC_UVE_ERROR,
job_template_fqname=job_template_fqname,
job_execution_id=job_execution_id,
exc_msg=repr(exp))
raise JobException(msg, job_execution_id)
self._logger.info("Device data json: " +
str(self._device_data))
playbook_process.wait(timeout=self._playbook_timeout)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
raise JobException(msg, self._execution_id)
def run_playbook(self, playbook_info):
try:
# create job log to capture the start of the playbook
device_id = \
playbook_info['extra_vars']['playbook_input'].get(
'device_id', "")
msg = MsgBundle.getMessage(
MsgBundle.START_EXECUTE_PLAYBOOK_MSG,
playbook_uri=playbook_info['uri'],
def read_device_data(self,
device_list,
request_params,
job_exec_id,
is_delete=False):
device_data = dict()
for device_id in device_list:
if not is_delete:
try:
(ok, result) = self.db_read("physical-router", device_id, [
'physical_router_user_credentials',
'physical_router_management_ip', 'fq_name',
'physical_router_device_family',
'physical_router_vendor_name',
'physical_router_product_name', 'fabric_refs'
])
if not ok:
msg = "Error while reading the physical router " \
"with id %s : %s" % (device_id, result)
raise JobException(msg, job_exec_id)
except NoIdError as ex:
msg = "Device not found" \
"%s: %s" % (device_id, str(ex))
raise JobException(msg, job_exec_id)
except Exception as e:
msg = "Exception while reading device %s %s " % \
(device_id, str(e))
raise JobException(msg, job_exec_id)
device_fq_name = result.get('fq_name')
device_mgmt_ip = result.get('physical_router_management_ip')
user_cred = result.get('physical_router_user_credentials')
device_family = result.get("physical_router_device_family")
device_vendor_name = result.get("physical_router_vendor_name")
device_product_name = result.get(
"physical_router_product_name")
fabric_refs = result.get('fabric_refs')
if fabric_refs:
fabric_fq_name = result.get('fabric_refs')[0].get('to')
fabric_fq_name_str = ':'.join(fabric_fq_name)
request_params['fabric_fq_name'] = fabric_fq_name_str
else:
device_mgmt_ip = request_params.get(
'input', {}).get('device_management_ip')
device_abs_cfg = request_params.get(
'input', {}).get('device_abstract_config')
system = device_abs_cfg.get('system', {})
device_name = system.get('name')
device_username = system.get('credentials',
{}).get('user_name')
device_password = system.get('credentials', {}).get('password')
user_cred = {
"username": device_username,
"password": device_password
}
device_family = system.get('device_family')
device_vendor_name = system.get('vendor_name')
device_product_name = system.get('product_name')
device_fq_name = ["default-global-system-config", device_name]
self.read_fabric_data(request_params, job_exec_id, is_delete)
device_json = {"device_management_ip": device_mgmt_ip}
device_json.update({"device_fqname": device_fq_name})
if user_cred:
device_json.update(
{"device_username": user_cred.get('username')})
device_json.update(
{"device_password": user_cred.get('password')})
if device_family:
device_json.update({"device_family": device_family})
if device_vendor_name:
device_json.update({"device_vendor": device_vendor_name})
if device_product_name:
device_json.update({"device_product": device_product_name})
device_data.update({device_id: device_json})
if len(device_data) > 0:
request_params.update({"device_json": device_data})
def get_playbook_info(self, job_percent_per_task, device_id=None):
try:
# create the cmd line param for the playbook
extra_vars = {
'input': self._job_input,
'prev_pb_output': self._playbook_output,
'params': self._job_params,
'job_template_id': self._job_template.get_uuid(),
'job_template_fqname': self._job_template.fq_name,
'fabric_fq_name': self._fabric_fq_name,
'auth_token': self._auth_token,
'job_execution_id': self._execution_id,
'args': self._sandesh_args,
'playbook_job_percentage': job_percent_per_task
}
playbooks = self._job_template.get_job_template_playbooks()
if device_id:
if not self._device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg, self._execution_id)
device_data = self._device_json.get(device_id)
if not device_data:
msg = MsgBundle.getMessage(MsgBundle.NO_DEVICE_DATA_FOUND,
device_id=device_id)
raise JobException(msg, self._execution_id)
device_family = device_data.get('device_family')
device_vendor = device_data.get('device_vendor')
if not device_vendor or not device_family:
msg = MsgBundle.getMessage(
MsgBundle.DEVICE_VENDOR_FAMILY_MISSING,
device_id=device_id)
raise JobException(msg, self._execution_id)
# check for credentials,required param; else playbooks
# will fail
device_username = device_data.get('device_username')
device_password = device_data.get('device_password')
if not device_username or not device_password:
msg = MsgBundle.getMessage(MsgBundle.NO_CREDENTIALS_FOUND,
device_id=device_id)
raise JobException(msg, self._execution_id)
# update extra-vars to reflect device-related params
device_fqname = device_data.get('device_fqname')
device_management_ip = device_data.get('device_management_ip')
extra_vars.update({
'device_id': device_id,
'device_fqname': device_fqname,
'device_management_ip': device_management_ip,
'vendor': device_vendor,
'device_family': device_family,
'device_username': device_username,
'device_password': device_password
})
self._logger.debug("Passing the following device "
"ip to playbook %s " % device_management_ip)
# get the playbook uri from the job template
play_info = playbooks.playbook_info[self._playbook_seq]
playbook_input = {'playbook_input': extra_vars}
playbook_info = dict()
playbook_info['uri'] = play_info.playbook_uri
playbook_info['extra_vars'] = playbook_input
return playbook_info
except JobException:
raise
except Exception as exp:
msg = MsgBundle.getMessage(
MsgBundle.GET_PLAYBOOK_INFO_ERROR,
job_template_id=self._job_template.get_uuid(),
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
def run_playbook_process(self, playbook_info, percentage_completed):
playbook_process = None
playbook_output = None
pr_uve_name = None
self.current_percentage = percentage_completed
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
unique_pb_id = str(uuid.uuid4())
playbook_info['extra_vars']['playbook_input']['unique_pb_id']\
= unique_pb_id
exec_id =\
playbook_info['extra_vars']['playbook_input'][
'job_execution_id']
device_fqname = \
playbook_info['extra_vars']['playbook_input'].get(
'device_fqname')
if device_fqname:
pr_fqname = ':'.join(map(str, device_fqname))
job_template_fq_name = ':'.join(
map(str, self._job_template.fq_name))
pr_uve_name = pr_fqname + ":" + \
self._fabric_fq_name + ":" + job_template_fq_name
pr_object_log_start_time = time.time()
playbook_process = subprocess32.Popen([
"python", playbook_exec_path, "-i",
json.dumps(playbook_info)
],
close_fds=True,
cwd='/')
# this is to yield the context to the playbooks so that
# they start running concurrently
gevent.sleep(0)
marked_output = self.process_file_and_get_marked_output(
unique_pb_id, exec_id, playbook_process, pr_uve_name)
marked_jsons = self._extract_marked_json(marked_output)
playbook_output = marked_jsons.get(PLAYBOOK_OUTPUT)
playbook_process.wait(timeout=self._playbook_timeout)
pr_object_log_end_time = time.time()
# create prouter UVE in job_manager only if it is not a multi
# device job template
if not self._job_template.get_job_template_multi_device_job():
self.send_prouter_uve(exec_id, pr_object_log_start_time,
pr_object_log_end_time,
playbook_process.returncode)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
except Exception as exp:
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_ERROR,
playbook_uri=playbook_info['uri'],
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
if playbook_process.returncode != 0:
msg = MsgBundle.getMessage(MsgBundle.PLAYBOOK_EXIT_WITH_ERROR,
playbook_uri=playbook_info['uri'])
if playbook_output:
msg = msg + "\n Error Message from playbook: %s" % playbook_output.get(
'message', "")
raise JobException(msg, self._execution_id)
return playbook_output
def start_job(self):
self._logger.info("Starting Executable")
job_error_msg = None
job_template = self.job_template
try:
# create job UVE and log
self.result_handler = JobResultHandler(self.job_template_id,
self.job_execution_id,
self.fabric_fq_name,
self._logger,
self.job_utils,
self.job_log_utils)
msg = MsgBundle.getMessage(MsgBundle.START_JOB_MESSAGE,
job_execution_id=self.job_execution_id,
job_template_name=\
job_template.fq_name[-1])
self._logger.debug(msg)
timestamp = int(round(time.time() * 1000))
self.job_log_utils.send_job_log(job_template.fq_name,
self.job_execution_id,
self.fabric_fq_name,
msg,
JobStatus.STARTING.value,
timestamp=timestamp)
# validate job input if required by job_template input_schema
input_schema = job_template.get_job_template_input_schema()
if input_schema:
self._validate_job_input(input_schema, self.job_data)
executable_list = job_template.get_job_template_executables()\
.get_executable_info()
for executable in executable_list:
exec_path = executable.get_executable_path()
exec_args = executable.get_executable_args()
job_input_args = self.gather_job_args()
try:
exec_process = subprocess32.Popen([exec_path,
"--job-input",
json.dumps(job_input_args),
'--debug', 'True'],
close_fds=True, cwd='/',
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE)
self.job_file_write.write_to_file(
self.job_execution_id,
"job_summary",
JobFileWrite.JOB_LOG,
{"job_status": "INPROGRESS"})
msg = "Child process pid = " + str(exec_process.pid)
self._logger.info(msg)
(out, err) = exec_process.communicate(timeout=self.executable_timeout)
self._logger.notice(str(out))
self._logger.notice(str(err))
except subprocess32.TimeoutExpired as timeout_exp:
if exec_process is not None:
os.kill(exec_process.pid, 9)
msg = MsgBundle.getMessage(
MsgBundle.RUN_EXECUTABLE_PROCESS_TIMEOUT,
exec_path=exec_path,
exc_msg=repr(timeout_exp))
raise JobException(msg, self.job_execution_id)
self._logger.info(exec_process.returncode)
self._logger.info("Executable Completed")
if exec_process.returncode != 0:
self.job_file_write.write_to_file(
self.job_execution_id,
"job_summary",
JobFileWrite.JOB_LOG,
{"job_status": "FAILED"})
msg = MsgBundle.getMessage(MsgBundle.
EXECUTABLE_RETURN_WITH_ERROR,
exec_uri=exec_path)
self._logger.error(msg)
else:
self.job_file_write.write_to_file(
self.job_execution_id,
"job_summary",
JobFileWrite.JOB_LOG,
{"job_status": "COMPLETED"})
except JobException as exp:
err_msg = "Job Exception recieved: %s " % repr(exp)
self._logger.error(err_msg)
self._logger.error("%s" % traceback.format_exc())
self.result_handler.update_job_status(JobStatus.FAILURE,
err_msg)
if job_template:
self.result_handler.create_job_summary_log(
job_template.fq_name)
job_error_msg = err_msg
except Exception as exp:
err_msg = "Error while executing job %s " % repr(exp)
self._logger.error(err_msg)
self._logger.error("%s" % traceback.format_exc())
self.result_handler.update_job_status(JobStatus.FAILURE,
err_msg)
self.result_handler.create_job_summary_log(job_template.fq_name)
job_error_msg = err_msg
finally:
# need to wait for the last job log and uve update to complete
# via sandesh and then close sandesh connection
sandesh_util = SandeshUtils(self._logger)
sandesh_util.close_sandesh_connection()
self._logger.info("Closed Sandesh connection")
if job_error_msg is not None:
sys.exit(job_error_msg)
def handle_job(self,
result_handler,
job_percent_per_task,
device_id=None,
device_name=None):
playbook_info = None
try:
msg = "Starting playbook execution for job template %s with " \
"execution id %s" % (self._job_template.get_uuid(),
self._execution_id)
self._logger.debug(msg)
# Always acquire the lock while executing the multi device jobs
if device_name is not None:
if not self._acquire_device_lock(device_name):
raise JobException(
MsgBundle.getMessage(MsgBundle.DEVICE_LOCK_FAILURE))
# get the playbook information from the job template
playbook_info = self.get_playbook_info(job_percent_per_task,
device_id)
# run the playbook and retrieve the playbook output if any
playbook_output = self.run_playbook(
playbook_info, result_handler.percentage_completed)
# retrieve the device_op_results in case it was set for
# generic device operations.
# retrieve the playbook output status for warning cases
playbook_output_results = None
playbook_output_status = ""
warning_msg = ""
if playbook_output:
playbook_output_results = playbook_output.get('results')
playbook_output_status = playbook_output.get('status')
result_handler.update_playbook_output(playbook_output)
warning_msg = playbook_output.get("message")
msg = MsgBundle.getMessage(
MsgBundle.PLAYBOOK_EXECUTION_COMPLETE,
job_template_name=self._job_template.get_fq_name()[-1],
job_execution_id=self._execution_id)
self._logger.debug(msg)
if playbook_output_status.lower() == "warning":
result_handler.update_job_status(
JobStatus.WARNING,
warning_msg,
device_id,
device_name,
pb_results=playbook_output_results)
else:
result_handler.update_job_status(
JobStatus.SUCCESS,
msg,
device_id,
device_name,
pb_results=playbook_output_results)
self.check_and_send_prouter_job_uve_for_multidevice(
playbook_info, JobStatus.SUCCESS.value,
playbook_output_results)
if self.current_percentage:
result_handler.percentage_completed = self.current_percentage
except JobException as job_exp:
self._logger.error("%s" % job_exp.msg)
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg,
device_id, device_name)
if playbook_info:
self.check_and_send_prouter_job_uve_for_multidevice(
playbook_info, JobStatus.FAILURE.value)
except Exception as exp:
self._logger.error("Error while executing job %s " % repr(exp))
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, exp.message,
device_id, device_name)
if playbook_info:
self.check_and_send_prouter_job_uve_for_multidevice(
playbook_info, JobStatus.FAILURE.value)
finally:
if device_name is not None:
self._release_device_lock(device_name)
class JobHandler(object):
def __init__(self, logger, vnc_api, job_template, execution_id, input,
params, job_utils, device_json, auth_token, job_log_utils,
sandesh_args, fabric_fq_name, playbook_timeout, playbook_seq):
self._logger = logger
self._vnc_api = vnc_api
self._job_template = job_template
self._execution_id = execution_id
self._job_input = input
self._job_params = params
self._job_utils = job_utils
self._device_json = device_json
self._auth_token = auth_token
self._job_log_utils = job_log_utils
self._sandesh_args = sandesh_args
self._fabric_fq_name = fabric_fq_name
self._playbook_timeout = playbook_timeout
self._playbook_seq = playbook_seq
self._device_data = None
def handle_job(self, result_handler, job_percent_per_task, device_id=None):
try:
msg = "Starting playbook execution for job template %s with " \
"execution id %s" % (self._job_template.get_uuid(),
self._execution_id)
self._logger.debug(msg)
# get the playbook information from the job template
playbook_info = self.get_playbook_info(job_percent_per_task,
device_id)
# run the playbook
self.run_playbook(playbook_info)
msg = MsgBundle.getMessage(
MsgBundle.PLAYBOOK_EXECUTION_COMPLETE,
job_template_id=self._job_template.get_uuid(),
job_execution_id=self._execution_id)
self._logger.debug(msg)
result_handler.update_job_status(JobStatus.SUCCESS, msg, device_id)
self.update_result_handler_device_data(result_handler)
except JobException as job_exp:
self._logger.error("%s" % job_exp.msg)
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, job_exp.msg,
device_id)
except Exception as exp:
self._logger.error("Error while executing job %s " % repr(exp))
self._logger.error("%s" % traceback.format_exc())
result_handler.update_job_status(JobStatus.FAILURE, exp.message,
device_id)
# def find_playbook_info(self, device_family, device_vendor, playbook_list):
#
# for playbook_info in playbook_list:
# pb_vendor_name = playbook_info.get_vendor()
# if not pb_vendor_name:
# # device_vendor agnostic
# return playbook_info
# if pb_vendor_name.lower() == device_vendor.lower():
# pb_device_family = playbook_info.get_device_family()
# if pb_device_family:
# if device_family.lower() == pb_device_family.lower():
# return playbook_info
# else:
# # device_family agnostic
# return playbook_info
# msg = MsgBundle.getMessage(MsgBundle.
# PLAYBOOK_INFO_DEVICE_MISMATCH,
# device_vendor=device_vendor,
# device_family=device_family)
# raise JobException(msg, self._execution_id)
def get_playbook_info(self, job_percent_per_task, device_id=None):
try:
# create the cmd line param for the playbook
extra_vars = {
'input': self._job_input,
'params': self._job_params,
'job_template_id': self._job_template.get_uuid(),
'job_template_fqname': self._job_template.fq_name,
'fabric_fq_name': self._fabric_fq_name,
'auth_token': self._auth_token,
'job_execution_id': self._execution_id,
'args': self._sandesh_args,
'playbook_job_percentage': job_percent_per_task
}
playbooks = self._job_template.get_job_template_playbooks()
if device_id:
if not self._device_json:
msg = MsgBundle.getMessage(MsgBundle.DEVICE_JSON_NOT_FOUND)
raise JobException(msg, self._execution_id)
device_data = self._device_json.get(device_id)
if not device_data:
msg = MsgBundle.getMessage(MsgBundle.NO_DEVICE_DATA_FOUND,
device_id=device_id)
raise JobException(msg, self._execution_id)
device_family = device_data.get('device_family')
device_vendor = device_data.get('device_vendor')
if not device_vendor or not device_family:
msg = MsgBundle.getMessage(
MsgBundle.DEVICE_VENDOR_FAMILY_MISSING,
device_id=device_id)
raise JobException(msg, self._execution_id)
# check for credentials,required param; else playbooks
# will fail
device_username = device_data.get('device_username')
device_password = device_data.get('device_password')
if not device_username or not device_password:
msg = MsgBundle.getMessage(MsgBundle.NO_CREDENTIALS_FOUND,
device_id=device_id)
raise JobException(msg, self._execution_id)
# update extra-vars to reflect device-related params
device_fqname = device_data.get('device_fqname')
device_management_ip = device_data.get('device_management_ip')
extra_vars.update({
'device_id': device_id,
'device_fqname': device_fqname,
'device_management_ip': device_management_ip,
'vendor': device_vendor,
'device_family': device_family,
'device_username': device_username,
'device_password': device_password
})
self._logger.debug("Passing the following device "
"ip to playbook %s " % device_management_ip)
# get the playbook uri from the job template
play_info = playbooks.playbook_info[self._playbook_seq]
playbook_input = {'playbook_input': extra_vars}
playbook_info = dict()
playbook_info['uri'] = play_info.playbook_uri
playbook_info['extra_vars'] = playbook_input
return playbook_info
except JobException:
raise
except Exception as exp:
msg = MsgBundle.getMessage(
MsgBundle.GET_PLAYBOOK_INFO_ERROR,
job_template_id=self._job_template.get_uuid(),
exc_msg=repr(exp))
raise JobException(msg, self._execution_id)
def run_playbook_process(self, playbook_info):
playbook_process = None
try:
playbook_exec_path = os.path.dirname(__file__) \
+ "/playbook_helper.py"
playbook_process = subprocess32.Popen([
"python", playbook_exec_path, "-i",
json.dumps(playbook_info)
],
close_fds=True,
cwd='/',
stdout=subprocess32.PIPE)
device_data_output = ""
device_data_keyword = 'DEVICEDATA##'
while True:
output = playbook_process.stdout.readline()
if output == '' and playbook_process.poll() is not None:
break
if output:
self._logger.debug(output)
# read device list data and store in variable result handler
if device_data_keyword in output:
device_data_output = output
if device_data_output != "":
device_data_output = self._extract_device_data_output(
device_data_output, device_data_keyword)
self._logger.info("Device data extracted from output" +
str(device_data_output))
try:
self._device_data = json.loads(device_data_output)
except ValueError, e:
self._device_data = ast.literal_eval(device_data_output)
self._logger.info("Device data json: " +
str(self._device_data))
playbook_process.wait(timeout=self._playbook_timeout)
except subprocess32.TimeoutExpired as timeout_exp:
if playbook_process is not None:
os.kill(playbook_process.pid, 9)
msg = MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT,
playbook_uri=playbook_info['uri'],
exc_msg=repr(timeout_exp))
raise JobException(msg, self._execution_id)
