def output(self, watcher_instance: WatcherModule,
watcher_result: WatcherResult):
data = watcher_result.to_dict()
if not self.include_state:
del data['state']
if not self.include_validations:
del data['failed_assertions']
data = self.__flatten(watcher_result.to_dict())
data.update(dict(tags='healthcheck'))
data.update(self.extra_fields)
self.gelf_logger.info(
'HealthcheckBot {}: Watcher {} - checks {}'.format(
self.get_application_manager().get_instance_settings().id,
watcher_instance.name,
'passed' if watcher_result.checks_passed else 'failed'),
extra=data)
def run_watcher(self, watcher: WatcherModule, trigger: TriggerModule) -> WatcherResult:
reporter = ValidationReporter(watcher, trigger)
try:
state = watcher.obtain_state(trigger)
except Exception as e:
raise WatcherRuntimeError(
'Watcher "{}" was unable to finish obtain state cycle: {}'.format(watcher.name, str(e)), e)
if watcher.enable_assertions:
try:
watcher.do_assertions(state, reporter)
for assertion in watcher.custom_assertions:
try:
assertion.do_assert(state, reporter, assertion.name)
except Exception as e:
reporter.error(assertion.name, 'Unexpected error: ' + str(e))
except Exception as e:
raise WatcherRuntimeError(
'Watcher "{}" was unable to run assertions check: {}'.format(watcher.name, str(e)), e)
try:
serialized_state = watcher.serialize_state(state)
watcher_result = WatcherResult(serialized_state, reporter.errors, reporter.extra_data)
except Exception as e:
raise WatcherRuntimeError(
'Watcher "{}" was unable serialize state: {}'.format(watcher.name, str(e)), e)
self.deliver_watcher_result(watcher, watcher_result)
return watcher_result
def step(self):
current_time = datetime.datetime.now()
for job in self.__jobs:
if job.next_run <= current_time:
self.logger.info("Running watcher {}".format(job.watcher.name))
try:
with time_limit(job.watcher.execution_timeout, msg="Execution watcher timeout"):
self.get_application_manager().run_watcher(job.watcher, self)
job.next_run = datetime.datetime.now() + datetime.timedelta(seconds=self.interval)
job.postpone_interval = None
job.fail_counter = 0
except Exception as e:
self.logger.error("Worker execution failed: " + str(e))
if job.postpone_interval is None:
job.postpone_interval = datetime.timedelta(seconds=self.interval)
job.postpone_interval *= 2
job.fail_counter += 1
if job.postpone_interval > self.__max_postpone_duration:
job.postpone_interval = self.__max_postpone_duration
job.next_run = datetime.datetime.now() + job.postpone_interval
self.logger.error(
"The next cycle will be postponed for {} seconds".format(job.postpone_interval.total_seconds())
)
watcher_result = WatcherResult({}, [ValidationError("execution_cycle", str(e), True)])
self.get_application_manager().deliver_watcher_result(job.watcher, watcher_result)
def output(self, watcher_instance: WatcherModule, watcher_result: WatcherResult):
data = watcher_result.to_dict()
if not self.include_state:
del data["state"]
if not self.include_validations:
del data["failed_assertions"]
data = self.__flatten(watcher_result.to_dict())
data.update(dict(tags="healthcheck", watcher_name=watcher_instance.name))
if self.include_instance_name:
data["instance"] = self.get_application_manager().get_instance_settings().id
if len(watcher_result.extra.keys()) > 0 and "extra" in data:
data.update(self.__flatten(watcher_result.extra))
del data["extra"]
data.update(self.extra_fields)
self.gelf_logger.info(
"HealthcheckBot {}: Watcher {} - checks {}".format(
self.get_application_manager().get_instance_settings().id,
watcher_instance.name,
"passed" if watcher_result.checks_passed else "failed",
),
extra=data,
)
def output(self, watcher_instance: WatcherModule,
watcher_result: WatcherResult):
self.target_logger.log(self.log_level, str(watcher_result.to_dict()))
def output(self, watcher_instance: WatcherModule,
watcher_result: WatcherResult):
print(watcher_result.to_dict())