def execute_scheduled_maintenance(self, maintenance_id):
LOG.debug("Maintenance id: {}".format(maintenance_id))
maintenance = models.Maintenance.objects.get(id=maintenance_id)
models.Maintenance.objects.filter(id=maintenance_id).update(
status=maintenance.RUNNING, started_at=datetime.now())
LOG.info("Maintenance {} is RUNNING".format(maintenance, ))
worker_name = get_worker_name()
task_history = TaskHistory.register(request=self.request,
worker_name=worker_name)
task_history.relevance = TaskHistory.RELEVANCE_CRITICAL
LOG.info("id: {} | task: {} | kwargs: {} | args: {}".format(
self.request.id, self.request.task, self.request.kwargs,
str(self.request.args)))
task_history.update_details(
persist=True, details="Executing Maintenance: {}".format(maintenance))
for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
# main_output = {}
hm.status = hm.RUNNING
hm.started_at = datetime.now()
hm.save()
if hm.host is None:
hm.status = hm.UNAVAILABLEHOST
hm.finished_at = datetime.now()
hm.save()
continue
host = hm.host
update_task = "\nRunning Maintenance on {}".format(host)
if maintenance.disable_alarms:
disable_alarms(hm.host)
param_dict = {}
params = models.MaintenanceParameters.objects.filter(
maintenance=maintenance)
for param in params:
param_function = get_function(param.function_name)
param_dict[param.parameter_name] = param_function(host.id)
main_script = build_context_script(param_dict, maintenance.main_script)
main_output = host.ssh.run_script(script=main_script,
raise_if_error=False)
if main_output['exit_code'] == 0:
hm.status = hm.SUCCESS
else:
if maintenance.rollback_script:
hm.status = hm.ROLLBACK
hm.save()
rollback_script = build_context_script(
param_dict, maintenance.rollback_script)
rollback_output = host.ssh.run_script(script=rollback_script,
raise_if_error=False)
if rollback_output['exit_code'] == 0:
hm.status = hm.ROLLBACK_SUCCESS
else:
hm.status = hm.ROLLBACK_ERROR
hm.rollback_log = get_dict_lines(rollback_output)
else:
hm.status = hm.ERROR
if maintenance.disable_alarms:
enable_alarms(hm.host)
update_task += "...status: {}".format(hm.status)
task_history.update_details(persist=True, details=update_task)
hm.main_log = get_dict_lines(main_output)
hm.finished_at = datetime.now()
hm.save()
models.Maintenance.objects.filter(id=maintenance_id).update(
status=maintenance.FINISHED, finished_at=datetime.now())
task_history.update_status_for(TaskHistory.STATUS_SUCCESS,
details='Maintenance executed succesfully')
LOG.info("Maintenance: {} has FINISHED".format(maintenance))
def execute_scheduled_maintenance(self, maintenance_id):
LOG.debug("Maintenance id: {}".format(maintenance_id))
maintenance = models.Maintenance.objects.get(id=maintenance_id)
models.Maintenance.objects.filter(id=maintenance_id).update(
status=maintenance.RUNNING, started_at=datetime.now()
)
LOG.info("Maintenance {} is RUNNING".format(maintenance,))
worker_name = get_worker_name()
task_history = TaskHistory.register(
request=self.request, worker_name=worker_name
)
task_history.relevance = TaskHistory.RELEVANCE_CRITICAL
LOG.info("id: {} | task: {} | kwargs: {} | args: {}".format(
self.request.id, self.request.task,
self.request.kwargs, str(self.request.args)
))
task_history.update_details(
persist=True, details="Executing Maintenance: {}".format(maintenance)
)
for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
main_output = {}
hm.status = hm.RUNNING
hm.started_at = datetime.now()
hm.save()
if hm.host is None:
hm.status = hm.UNAVAILABLEHOST
hm.finished_at = datetime.now()
hm.save()
continue
host = hm.host
update_task = "\nRunning Maintenance on {}".format(host)
if maintenance.disable_alarms:
disable_alarms(hm.host)
param_dict = {}
params = models.MaintenanceParameters.objects.filter(
maintenance=maintenance
)
for param in params:
param_function = get_function(param.function_name)
param_dict[param.parameter_name] = param_function(host.id)
main_script = build_context_script(param_dict, maintenance.main_script)
exit_status = exec_remote_command_host(host, main_script, main_output)
if exit_status == 0:
hm.status = hm.SUCCESS
else:
if maintenance.rollback_script:
rollback_output = {}
hm.status = hm.ROLLBACK
hm.save()
rollback_script = build_context_script(
param_dict, maintenance.rollback_script
)
exit_status = exec_remote_command_host(
host, rollback_script, rollback_output
)
if exit_status == 0:
hm.status = hm.ROLLBACK_SUCCESS
else:
hm.status = hm.ROLLBACK_ERROR
hm.rollback_log = get_dict_lines(rollback_output)
else:
hm.status = hm.ERROR
if maintenance.disable_alarms:
enable_alarms(hm.host)
update_task += "...status: {}".format(hm.status)
task_history.update_details(persist=True, details=update_task)
hm.main_log = get_dict_lines(main_output)
hm.finished_at = datetime.now()
hm.save()
models.Maintenance.objects.filter(id=maintenance_id).update(
status=maintenance.FINISHED, finished_at=datetime.now()
)
task_history.update_status_for(
TaskHistory.STATUS_SUCCESS, details='Maintenance executed succesfully'
)
LOG.info("Maintenance: {} has FINISHED".format(maintenance))