class LoggerOutput(OutputModule):
def __init__(self, application):
super().__init__(application)
self.logger_name = None # type: str
self.log_level = logging.INFO # type: int
self.include_state = True
self.include_validations = True
self.target_logger = None # type: logging.Logger
def output(self, watcher_instance: WatcherModule,
watcher_result: WatcherResult):
self.target_logger.log(self.log_level, str(watcher_result.to_dict()))
def on_configured(self):
self.target_logger = logging.getLogger(self.logger_name)
self.target_logger.setLevel(self.log_level)
PARAMS = (
ParameterDef('logger_name', is_required=True),
ParameterDef('log_level',
sanitize_fn=lambda level_name: logging._nameToLevel.get(
level_name.upper()),
validators=(validators.integer, )),
ParameterDef('include_instance_name',
validators=(validators.boolean, )),
ParameterDef('include_state', validators=(validators.boolean, )),
ParameterDef('include_validations', validators=(validators.boolean, )),
)
class DatabaseQueryWatcher(WatcherModule):
PARAMS = [
ParameterDef('db_connection', is_required=True),
ParameterDef('queries', is_required=True),
]
def __init__(self, application):
super().__init__(application)
self.db_connection = {}
self.queries = {}
def serialize_state(self, state: object) -> [dict, None]:
return json.dumps(state, cls=DecimalEncoder)
def on_configured(self):
self.conn = psycopg2.connect(**self.db_connection, cursor_factory=RealDictCursor)
def obtain_state(self, trigger) -> object:
cur = self.conn.cursor()
try:
result = {}
for name, query in self.queries.items():
result[name] = []
cur.execute(query)
result[name] = [dict(x) for x in cur.fetchall()]
return result
finally:
cur.close()
class HttpRequest(WatcherModule):
def __init__(self, application):
super().__init__(application)
self.url = None
self.headers = None
self.auth_user = None
self.auth_password = None
self.payload = None
self.method = 'GET'
self.timeout = 4
self.assert_status = None
self.assert_response_time = None
def obtain_state(self, trigger):
return requests.request(self.method,
self.url,
data=self.payload,
headers=self.headers,
auth=self.basic_auth,
timeout=self.timeout)
@property
def basic_auth(self):
return None if self.auth_user is None and self.auth_password is None else (
self.auth_user, self.auth_password)
def serialize_state(self, state: requests.Response):
return dict(status_code=state.status_code,
response_time=state.elapsed.total_seconds(),
url=state.url,
redirect_history=[
dict(status_code=x.status_code, url=x.url)
for x in state.history
])
def do_assertions(self, state: requests.Response,
reporter: ValidationReporter):
if self.assert_status is not None and state.status_code != self.assert_status:
reporter.error(
'status_code', 'Expected HTTP status {} but got {}'.format(
self.assert_status, state.status_code))
if self.assert_response_time is not None and state.elapsed.total_seconds(
) > self.assert_response_time:
reporter.error(
'response_time', 'Expected HTTP response time must be '
'< {} but actual is {}'.format(self.assert_response_time,
state.elapsed.total_seconds()))
PARAMS = (
ParameterDef('url', is_required=True,
validators=(validators.string, )),
ParameterDef('method', validators=(validators.string, )),
ParameterDef('auth_user', validators=(validators.string, )),
ParameterDef('auth_password', validators=(validators.string, )),
ParameterDef('headers', validators=(validators.dict_of_strings, )),
ParameterDef('timeout', validators=(validators.integer, )),
ParameterDef('payload'),
ParameterDef('assert_status'),
ParameterDef('assert_response_time', validators=(validators.number, )),
)
class SimpleTimer(TriggerModule, LoopModuleMixin):
def __init__(self, application):
super().__init__(application)
self.interval = 300
self.start_immediately = True
self.__jobs = [] # type: List[SimpleTimerJob]
self.__max_postpone_duration = datetime.timedelta(minutes=20)
self.__postpone_interval = None
def on_configured(self):
pass
def register_watcher(self, watcher: WatcherModule):
self.__jobs.append(
SimpleTimerJob(
next_run=datetime.datetime.now()
+ datetime.timedelta(seconds=0 if self.start_immediately else self.interval),
watcher=watcher,
)
)
def step(self):
current_time = datetime.datetime.now()
for job in self.__jobs:
if job.next_run <= current_time:
self.logger.info("Running watcher {}".format(job.watcher.name))
try:
with time_limit(job.watcher.execution_timeout, msg="Execution watcher timeout"):
self.get_application_manager().run_watcher(job.watcher, self)
job.next_run = datetime.datetime.now() + datetime.timedelta(seconds=self.interval)
job.postpone_interval = None
job.fail_counter = 0
except Exception as e:
self.logger.error("Worker execution failed: " + str(e))
if job.postpone_interval is None:
job.postpone_interval = datetime.timedelta(seconds=self.interval)
job.postpone_interval *= 2
job.fail_counter += 1
if job.postpone_interval > self.__max_postpone_duration:
job.postpone_interval = self.__max_postpone_duration
job.next_run = datetime.datetime.now() + job.postpone_interval
self.logger.error(
"The next cycle will be postponed for {} seconds".format(job.postpone_interval.total_seconds())
)
watcher_result = WatcherResult({}, [ValidationError("execution_cycle", str(e), True)])
self.get_application_manager().deliver_watcher_result(job.watcher, watcher_result)
PARAMS = (
ParameterDef("interval", validators=(validators.integer,)),
ParameterDef("start_immediately", validators=(validators.boolean,)),
)
class RedisQueueSizeWatcher(WatcherModule):
def __init__(self, application):
super().__init__(application)
self.url = None
self.host = 'localhost'
self.port = 6379
self.db = 0
self.queue_name = 'celery'
self.assert_min_qty = None
self.assert_max_qty = None
self.__redis = None # type: redis.Redis
self.socket_timeout = 7000
def obtain_state(self, trigger) -> object:
if self.__redis is None:
if self.url:
self.__redis = redis.Redis.from_url(url=self.url, db=self.db, socket_timeout=self.socket_timeout,
socket_connect_timeout=self.socket_timeout, max_connections=1)
else:
self.__redis = redis.Redis(host=self.host, port=self.port, db=self.db,
socket_timeout=self.socket_timeout, max_connections=1,
socket_connect_timeout=self.socket_timeout)
messages_count = self.__redis.llen(self.queue_name)
self.__redis.connection_pool.disconnect()
return messages_count
def serialize_state(self, state: int) -> [dict, None]:
return {
"queue": self.queue_name,
"msg_qty": state
}
def do_assertions(self, state: int, reporter: ValidationReporter):
if self.assert_min_qty is not None:
if state < self.assert_min_qty:
reporter.error('min_qty_assert', "Expected minimum number of messages in queue is {} but actual qty "
"is {}".format(self.assert_min_qty, state))
if self.assert_max_qty is not None:
if state > self.assert_max_qty:
reporter.error('max_qty_assert', "Expected maximum number of messages in queue is {} but actual qty "
"is {}".format(self.assert_max_qty, state))
PARAMS = (
ParameterDef('url', validators=(validators.string,)),
ParameterDef('host', validators=(validators.string,)),
ParameterDef('queue_name', validators=(validators.string,)),
ParameterDef('port', validators=(validators.integer,)),
ParameterDef('db', validators=(validators.integer,)),
ParameterDef('assert_min_qty', validators=(validators.integer,)),
ParameterDef('assert_max_qty', validators=(validators.integer,)),
)
class TitleAssert(WatcherAssert):
def __init__(self, application):
super().__init__(application)
self.expected_title = None
def do_assert(self, state: requests.Response, reporter: ValidationReporter,
assertion_name: str):
soup = BeautifulSoup(state.content, 'html.parser')
actual_title = soup.title.string if soup.title else None
if actual_title != self.expected_title:
reporter.error(
assertion_name,
'Expected title is "{}" but actual is "{}"'.format(
self.expected_title, actual_title))
PARAMS = (ParameterDef('expected_title',
is_required=True,
validators=(validators.string, )), )
class SystemTimeWatcher(WatcherModule):
def __init__(self, application):
super().__init__(application)
self.error_when_midnight = False
def obtain_state(self, trigger) -> object:
current_time = datetime.now()
return current_time
def serialize_state(self, state: datetime) -> [dict, None]:
return {"time": state.isoformat()}
def do_assertions(self, state: datetime, reporter: ValidationReporter):
if self.error_when_midnight:
if state.time() == time(0, 0):
reporter.error('its_midnight')
PARAMS = (ParameterDef('error_when_midnight',
validators=(validators.boolean, )), )
class GelfOutput(OutputModule):
def __init__(self, application):
super().__init__(application)
self.gelf_logger = None # type: logging.Logger
self.gelf_port = 9402 # type: int
self.gelf_host = None # type: str
self.gelf_protocol = 'udp'
self.include_state = True
self.include_validations = True
self.extra_fields = {}
def __flatten(self, dictionary, parent_key='', sep='__'):
items = []
for k, v in dictionary.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(self.__flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
def on_configured(self):
self.gelf_logger = logging.getLogger('GELF')
self.gelf_logger.setLevel(logging.DEBUG)
self.gelf_logger.propagate = False
if self.gelf_protocol == 'udp':
self.gelf_logger.addHandler(
GELFHandler(host=self.gelf_host,
port=self.gelf_port,
debugging_fields=False,
extra_fields=True))
elif self.gelf_protocol == 'tcp':
handler = GELFTcpHandler(host=self.gelf_host,
port=self.gelf_port,
debugging_fields=False,
extra_fields=True)
handler.level = logging.DEBUG
self.gelf_logger.addHandler(handler)
else:
raise ConfigValidationError(
'/'.join(('watchers', self.name)),
"Parameter gelf_protocol must be one of: udp, tcp but {} given"
.format(self.gelf_protocol))
def output(self, watcher_instance: WatcherModule,
watcher_result: WatcherResult):
data = watcher_result.to_dict()
if not self.include_state:
del data['state']
if not self.include_validations:
del data['failed_assertions']
data = self.__flatten(watcher_result.to_dict())
data.update(dict(tags='healthcheck'))
data.update(self.extra_fields)
self.gelf_logger.info(
'HealthcheckBot {}: Watcher {} - checks {}'.format(
self.get_application_manager().get_instance_settings().id,
watcher_instance.name,
'passed' if watcher_result.checks_passed else 'failed'),
extra=data)
PARAMS = (
ParameterDef('gelf_host', is_required=True),
ParameterDef('gelf_port', validators=(validators.integer, )),
ParameterDef('gelf_protocol', validators=(validators.string, )),
ParameterDef('include_state', validators=(validators.boolean, )),
ParameterDef('include_validations', validators=(validators.boolean, )),
ParameterDef('extra_fields',
validators=(validators.dict_of_strings, )),
)
class GelfOutput(OutputModule):
def __init__(self, application):
super().__init__(application)
self.gelf_logger = None # type: logging.Logger
self.gelf_port = 9402 # type: int
self.gelf_host = None # type: str
self.gelf_protocol = "udp"
self.facility = "healthcheck"
self.include_state = True
self.include_validations = True
self.include_instance_name = True
self.extra_fields = {}
def __flatten(self, dictionary, parent_key="", sep="__"):
items = []
for k, v in dictionary.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(self.__flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
def on_configured(self):
self.gelf_logger = logging.getLogger("GELF")
self.gelf_logger.setLevel(logging.DEBUG)
self.gelf_logger.propagate = False
if self.gelf_protocol == "udp":
self.gelf_logger.addHandler(
GELFHandler(
host=self.gelf_host,
port=self.gelf_port,
facility=self.facility,
debugging_fields=False,
extra_fields=True,
)
)
elif self.gelf_protocol == "tcp":
handler = GELFTcpHandler(
host=self.gelf_host,
facility=self.facility,
port=self.gelf_port,
debugging_fields=False,
extra_fields=True,
)
handler.level = logging.DEBUG
self.gelf_logger.addHandler(handler)
else:
raise ConfigValidationError(
"/".join(("watchers", self.name)),
"Parameter gelf_protocol must be one of: udp, tcp but {} given".format(self.gelf_protocol),
)
def output(self, watcher_instance: WatcherModule, watcher_result: WatcherResult):
data = watcher_result.to_dict()
if not self.include_state:
del data["state"]
if not self.include_validations:
del data["failed_assertions"]
data = self.__flatten(watcher_result.to_dict())
data.update(dict(tags="healthcheck", watcher_name=watcher_instance.name))
if self.include_instance_name:
data["instance"] = self.get_application_manager().get_instance_settings().id
if len(watcher_result.extra.keys()) > 0 and "extra" in data:
data.update(self.__flatten(watcher_result.extra))
del data["extra"]
data.update(self.extra_fields)
self.gelf_logger.info(
"HealthcheckBot {}: Watcher {} - checks {}".format(
self.get_application_manager().get_instance_settings().id,
watcher_instance.name,
"passed" if watcher_result.checks_passed else "failed",
),
extra=data,
)
PARAMS = (
ParameterDef("gelf_host", is_required=True),
ParameterDef("gelf_port", validators=(validators.integer,)),
ParameterDef("gelf_protocol", validators=(validators.string,)),
ParameterDef("facility", validators=(validators.string,)),
ParameterDef("include_state", validators=(validators.boolean,)),
ParameterDef("include_validations", validators=(validators.boolean,)),
ParameterDef("extra_fields", validators=(validators.dict_of_strings,)),
)
