class LoggerOutput(OutputModule): def __init__(self, application): super().__init__(application) self.logger_name = None # type: str self.log_level = logging.INFO # type: int self.include_state = True self.include_validations = True self.target_logger = None # type: logging.Logger def output(self, watcher_instance: WatcherModule, watcher_result: WatcherResult): self.target_logger.log(self.log_level, str(watcher_result.to_dict())) def on_configured(self): self.target_logger = logging.getLogger(self.logger_name) self.target_logger.setLevel(self.log_level) PARAMS = ( ParameterDef('logger_name', is_required=True), ParameterDef('log_level', sanitize_fn=lambda level_name: logging._nameToLevel.get( level_name.upper()), validators=(validators.integer, )), ParameterDef('include_instance_name', validators=(validators.boolean, )), ParameterDef('include_state', validators=(validators.boolean, )), ParameterDef('include_validations', validators=(validators.boolean, )), )
class DatabaseQueryWatcher(WatcherModule): PARAMS = [ ParameterDef('db_connection', is_required=True), ParameterDef('queries', is_required=True), ] def __init__(self, application): super().__init__(application) self.db_connection = {} self.queries = {} def serialize_state(self, state: object) -> [dict, None]: return json.dumps(state, cls=DecimalEncoder) def on_configured(self): self.conn = psycopg2.connect(**self.db_connection, cursor_factory=RealDictCursor) def obtain_state(self, trigger) -> object: cur = self.conn.cursor() try: result = {} for name, query in self.queries.items(): result[name] = [] cur.execute(query) result[name] = [dict(x) for x in cur.fetchall()] return result finally: cur.close()
class HttpRequest(WatcherModule): def __init__(self, application): super().__init__(application) self.url = None self.headers = None self.auth_user = None self.auth_password = None self.payload = None self.method = 'GET' self.timeout = 4 self.assert_status = None self.assert_response_time = None def obtain_state(self, trigger): return requests.request(self.method, self.url, data=self.payload, headers=self.headers, auth=self.basic_auth, timeout=self.timeout) @property def basic_auth(self): return None if self.auth_user is None and self.auth_password is None else ( self.auth_user, self.auth_password) def serialize_state(self, state: requests.Response): return dict(status_code=state.status_code, response_time=state.elapsed.total_seconds(), url=state.url, redirect_history=[ dict(status_code=x.status_code, url=x.url) for x in state.history ]) def do_assertions(self, state: requests.Response, reporter: ValidationReporter): if self.assert_status is not None and state.status_code != self.assert_status: reporter.error( 'status_code', 'Expected HTTP status {} but got {}'.format( self.assert_status, state.status_code)) if self.assert_response_time is not None and state.elapsed.total_seconds( ) > self.assert_response_time: reporter.error( 'response_time', 'Expected HTTP response time must be ' '< {} but actual is {}'.format(self.assert_response_time, state.elapsed.total_seconds())) PARAMS = ( ParameterDef('url', is_required=True, validators=(validators.string, )), ParameterDef('method', validators=(validators.string, )), ParameterDef('auth_user', validators=(validators.string, )), ParameterDef('auth_password', validators=(validators.string, )), ParameterDef('headers', validators=(validators.dict_of_strings, )), ParameterDef('timeout', validators=(validators.integer, )), ParameterDef('payload'), ParameterDef('assert_status'), ParameterDef('assert_response_time', validators=(validators.number, )), )
class SimpleTimer(TriggerModule, LoopModuleMixin): def __init__(self, application): super().__init__(application) self.interval = 300 self.start_immediately = True self.__jobs = [] # type: List[SimpleTimerJob] self.__max_postpone_duration = datetime.timedelta(minutes=20) self.__postpone_interval = None def on_configured(self): pass def register_watcher(self, watcher: WatcherModule): self.__jobs.append( SimpleTimerJob( next_run=datetime.datetime.now() + datetime.timedelta(seconds=0 if self.start_immediately else self.interval), watcher=watcher, ) ) def step(self): current_time = datetime.datetime.now() for job in self.__jobs: if job.next_run <= current_time: self.logger.info("Running watcher {}".format(job.watcher.name)) try: with time_limit(job.watcher.execution_timeout, msg="Execution watcher timeout"): self.get_application_manager().run_watcher(job.watcher, self) job.next_run = datetime.datetime.now() + datetime.timedelta(seconds=self.interval) job.postpone_interval = None job.fail_counter = 0 except Exception as e: self.logger.error("Worker execution failed: " + str(e)) if job.postpone_interval is None: job.postpone_interval = datetime.timedelta(seconds=self.interval) job.postpone_interval *= 2 job.fail_counter += 1 if job.postpone_interval > self.__max_postpone_duration: job.postpone_interval = self.__max_postpone_duration job.next_run = datetime.datetime.now() + job.postpone_interval self.logger.error( "The next cycle will be postponed for {} seconds".format(job.postpone_interval.total_seconds()) ) watcher_result = WatcherResult({}, [ValidationError("execution_cycle", str(e), True)]) self.get_application_manager().deliver_watcher_result(job.watcher, watcher_result) PARAMS = ( ParameterDef("interval", validators=(validators.integer,)), ParameterDef("start_immediately", validators=(validators.boolean,)), )
class RedisQueueSizeWatcher(WatcherModule): def __init__(self, application): super().__init__(application) self.url = None self.host = 'localhost' self.port = 6379 self.db = 0 self.queue_name = 'celery' self.assert_min_qty = None self.assert_max_qty = None self.__redis = None # type: redis.Redis self.socket_timeout = 7000 def obtain_state(self, trigger) -> object: if self.__redis is None: if self.url: self.__redis = redis.Redis.from_url(url=self.url, db=self.db, socket_timeout=self.socket_timeout, socket_connect_timeout=self.socket_timeout, max_connections=1) else: self.__redis = redis.Redis(host=self.host, port=self.port, db=self.db, socket_timeout=self.socket_timeout, max_connections=1, socket_connect_timeout=self.socket_timeout) messages_count = self.__redis.llen(self.queue_name) self.__redis.connection_pool.disconnect() return messages_count def serialize_state(self, state: int) -> [dict, None]: return { "queue": self.queue_name, "msg_qty": state } def do_assertions(self, state: int, reporter: ValidationReporter): if self.assert_min_qty is not None: if state < self.assert_min_qty: reporter.error('min_qty_assert', "Expected minimum number of messages in queue is {} but actual qty " "is {}".format(self.assert_min_qty, state)) if self.assert_max_qty is not None: if state > self.assert_max_qty: reporter.error('max_qty_assert', "Expected maximum number of messages in queue is {} but actual qty " "is {}".format(self.assert_max_qty, state)) PARAMS = ( ParameterDef('url', validators=(validators.string,)), ParameterDef('host', validators=(validators.string,)), ParameterDef('queue_name', validators=(validators.string,)), ParameterDef('port', validators=(validators.integer,)), ParameterDef('db', validators=(validators.integer,)), ParameterDef('assert_min_qty', validators=(validators.integer,)), ParameterDef('assert_max_qty', validators=(validators.integer,)), )
class TitleAssert(WatcherAssert): def __init__(self, application): super().__init__(application) self.expected_title = None def do_assert(self, state: requests.Response, reporter: ValidationReporter, assertion_name: str): soup = BeautifulSoup(state.content, 'html.parser') actual_title = soup.title.string if soup.title else None if actual_title != self.expected_title: reporter.error( assertion_name, 'Expected title is "{}" but actual is "{}"'.format( self.expected_title, actual_title)) PARAMS = (ParameterDef('expected_title', is_required=True, validators=(validators.string, )), )
class SystemTimeWatcher(WatcherModule): def __init__(self, application): super().__init__(application) self.error_when_midnight = False def obtain_state(self, trigger) -> object: current_time = datetime.now() return current_time def serialize_state(self, state: datetime) -> [dict, None]: return {"time": state.isoformat()} def do_assertions(self, state: datetime, reporter: ValidationReporter): if self.error_when_midnight: if state.time() == time(0, 0): reporter.error('its_midnight') PARAMS = (ParameterDef('error_when_midnight', validators=(validators.boolean, )), )
class GelfOutput(OutputModule): def __init__(self, application): super().__init__(application) self.gelf_logger = None # type: logging.Logger self.gelf_port = 9402 # type: int self.gelf_host = None # type: str self.gelf_protocol = 'udp' self.include_state = True self.include_validations = True self.extra_fields = {} def __flatten(self, dictionary, parent_key='', sep='__'): items = [] for k, v in dictionary.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, collections.MutableMapping): items.extend(self.__flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items) def on_configured(self): self.gelf_logger = logging.getLogger('GELF') self.gelf_logger.setLevel(logging.DEBUG) self.gelf_logger.propagate = False if self.gelf_protocol == 'udp': self.gelf_logger.addHandler( GELFHandler(host=self.gelf_host, port=self.gelf_port, debugging_fields=False, extra_fields=True)) elif self.gelf_protocol == 'tcp': handler = GELFTcpHandler(host=self.gelf_host, port=self.gelf_port, debugging_fields=False, extra_fields=True) handler.level = logging.DEBUG self.gelf_logger.addHandler(handler) else: raise ConfigValidationError( '/'.join(('watchers', self.name)), "Parameter gelf_protocol must be one of: udp, tcp but {} given" .format(self.gelf_protocol)) def output(self, watcher_instance: WatcherModule, watcher_result: WatcherResult): data = watcher_result.to_dict() if not self.include_state: del data['state'] if not self.include_validations: del data['failed_assertions'] data = self.__flatten(watcher_result.to_dict()) data.update(dict(tags='healthcheck')) data.update(self.extra_fields) self.gelf_logger.info( 'HealthcheckBot {}: Watcher {} - checks {}'.format( self.get_application_manager().get_instance_settings().id, watcher_instance.name, 'passed' if watcher_result.checks_passed else 'failed'), extra=data) PARAMS = ( ParameterDef('gelf_host', is_required=True), ParameterDef('gelf_port', validators=(validators.integer, )), ParameterDef('gelf_protocol', validators=(validators.string, )), ParameterDef('include_state', validators=(validators.boolean, )), ParameterDef('include_validations', validators=(validators.boolean, )), ParameterDef('extra_fields', validators=(validators.dict_of_strings, )), )
class GelfOutput(OutputModule): def __init__(self, application): super().__init__(application) self.gelf_logger = None # type: logging.Logger self.gelf_port = 9402 # type: int self.gelf_host = None # type: str self.gelf_protocol = "udp" self.facility = "healthcheck" self.include_state = True self.include_validations = True self.include_instance_name = True self.extra_fields = {} def __flatten(self, dictionary, parent_key="", sep="__"): items = [] for k, v in dictionary.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, collections.MutableMapping): items.extend(self.__flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items) def on_configured(self): self.gelf_logger = logging.getLogger("GELF") self.gelf_logger.setLevel(logging.DEBUG) self.gelf_logger.propagate = False if self.gelf_protocol == "udp": self.gelf_logger.addHandler( GELFHandler( host=self.gelf_host, port=self.gelf_port, facility=self.facility, debugging_fields=False, extra_fields=True, ) ) elif self.gelf_protocol == "tcp": handler = GELFTcpHandler( host=self.gelf_host, facility=self.facility, port=self.gelf_port, debugging_fields=False, extra_fields=True, ) handler.level = logging.DEBUG self.gelf_logger.addHandler(handler) else: raise ConfigValidationError( "/".join(("watchers", self.name)), "Parameter gelf_protocol must be one of: udp, tcp but {} given".format(self.gelf_protocol), ) def output(self, watcher_instance: WatcherModule, watcher_result: WatcherResult): data = watcher_result.to_dict() if not self.include_state: del data["state"] if not self.include_validations: del data["failed_assertions"] data = self.__flatten(watcher_result.to_dict()) data.update(dict(tags="healthcheck", watcher_name=watcher_instance.name)) if self.include_instance_name: data["instance"] = self.get_application_manager().get_instance_settings().id if len(watcher_result.extra.keys()) > 0 and "extra" in data: data.update(self.__flatten(watcher_result.extra)) del data["extra"] data.update(self.extra_fields) self.gelf_logger.info( "HealthcheckBot {}: Watcher {} - checks {}".format( self.get_application_manager().get_instance_settings().id, watcher_instance.name, "passed" if watcher_result.checks_passed else "failed", ), extra=data, ) PARAMS = ( ParameterDef("gelf_host", is_required=True), ParameterDef("gelf_port", validators=(validators.integer,)), ParameterDef("gelf_protocol", validators=(validators.string,)), ParameterDef("facility", validators=(validators.string,)), ParameterDef("include_state", validators=(validators.boolean,)), ParameterDef("include_validations", validators=(validators.boolean,)), ParameterDef("extra_fields", validators=(validators.dict_of_strings,)), )