def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbitmq = RabbitMQApi(
self.dummy_logger, env.RABBIT_IP,
connection_check_time_interval=self.connection_check_time_interval)
self.manager_name = 'test_github_alerters_manager'
self.test_queue_name = 'Test Queue'
self.test_data_str = 'test data'
self.test_heartbeat = {
'component_name': self.manager_name,
'is_alive': True,
'timestamp': datetime(2012, 1, 1).timestamp(),
}
self.github_alerter_name = GITHUB_ALERTER_NAME
self.dummy_process1 = Process(target=infinite_fn, args=())
self.dummy_process1.daemon = True
self.dummy_process2 = Process(target=infinite_fn, args=())
self.dummy_process2.daemon = True
self.dummy_process3 = Process(target=infinite_fn, args=())
self.dummy_process3.daemon = True
self.test_rabbit_manager = RabbitMQApi(
self.dummy_logger, env.RABBIT_IP,
connection_check_time_interval=self.connection_check_time_interval)
self.test_manager = GithubAlerterManager(
self.dummy_logger, self.manager_name, self.rabbitmq)
self.test_exception = PANICException('test_exception', 1)
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.manager_name = 'test_data_transformers_manager'
self.test_queue_name = 'Test Queue'
self.test_data_str = 'test data'
self.test_timestamp = datetime(2012, 1, 1).timestamp()
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.dummy_process1 = Process(target=infinite_fn, args=())
self.dummy_process1.daemon = True
self.dummy_process2 = Process(target=infinite_fn, args=())
self.dummy_process2.daemon = True
self.dummy_process3 = Process(target=infinite_fn, args=())
self.dummy_process3.daemon = True
self.transformer_process_dict_example = {
SYSTEM_DATA_TRANSFORMER_NAME: self.dummy_process1,
GITHUB_DATA_TRANSFORMER_NAME: self.dummy_process2,
}
self.test_manager = DataTransformersManager(self.dummy_logger,
self.manager_name,
self.rabbitmq)
self.test_exception = PANICException('test_exception', 1)
def delete_exchange_if_exists(rabbit: RabbitMQApi, exchange_name: str) -> None:
try:
rabbit.exchange_declare(exchange_name, passive=True)
rabbit.exchange_delete(exchange_name)
except pika.exceptions.ChannelClosedByBroker:
print("Exchange {} does not exist - don't need to close".format(
exchange_name))
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.monitor_name = 'test_monitor'
self.monitoring_period = 10
self.repo_id = 'test_repo_id'
self.parent_id = 'test_parent_id'
self.repo_name = 'test_repo'
self.monitor_repo = True
self.releases_page = 'test_url'
self.routing_key = 'test_routing_key'
self.test_data_str = 'test data'
self.test_data_dict = {
'test_key_1': 'test_val_1',
'test_key_2': 'test_val_2',
}
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': datetime(2012, 1, 1).timestamp(),
}
self.test_queue_name = 'Test Queue'
# In the real retrieved data there are more fields, but these are the
# only ones that interest us so far.
self.retrieved_metrics_example = [
{
'name': 'First Release 😮',
'tag_name': 'v1.0.0'
},
{
'name': 'Release Candidate 1',
'tag_name': 'v0.1.0'
},
]
self.processed_data_example = {
'0': {
'release_name': 'First Release 😮',
'tag_name': 'v1.0.0'
},
'1': {
'release_name': 'Release Candidate 1',
'tag_name': 'v0.1.0'
},
}
self.test_exception = PANICException('test_exception', 1)
self.repo_config = RepoConfig(self.repo_id, self.parent_id,
self.repo_name, self.monitor_repo,
self.releases_page)
self.test_monitor = GitHubMonitor(self.monitor_name, self.repo_config,
self.dummy_logger,
self.monitoring_period,
self.rabbitmq)
def _initialise_telegram_commands_handler(
bot_token: str, bot_chat_id: str, channel_id: str, channel_name: str,
associated_chains: Dict) -> TelegramCommandsHandler:
# Handler display name based on channel name
handler_display_name = TELEGRAM_COMMANDS_HANDLER_NAME_TEMPLATE.format(
channel_name)
handler_logger = _initialise_channel_handler_logger(
handler_display_name, TelegramCommandsHandler.__name__)
# Try initialising handler until successful
while True:
try:
telegram_bot = TelegramBotApi(bot_token, bot_chat_id)
telegram_channel = TelegramChannel(
channel_name, channel_id,
handler_logger.getChild(TelegramChannel.__name__),
telegram_bot)
cmd_handlers_logger = handler_logger.getChild(
TelegramCommandHandlers.__name__)
cmd_handlers_rabbitmq = RabbitMQApi(
logger=cmd_handlers_logger.getChild(RabbitMQApi.__name__),
host=env.RABBIT_IP)
cmd_handlers_redis = RedisApi(
logger=cmd_handlers_logger.getChild(RedisApi.__name__),
host=env.REDIS_IP,
db=env.REDIS_DB,
port=env.REDIS_PORT,
namespace=env.UNIQUE_ALERTER_IDENTIFIER)
cmd_handlers_mongo = MongoApi(logger=cmd_handlers_logger.getChild(
MongoApi.__name__),
host=env.DB_IP,
db_name=env.DB_NAME,
port=env.DB_PORT)
cmd_handlers = TelegramCommandHandlers(
TELEGRAM_COMMAND_HANDLERS_NAME, cmd_handlers_logger,
associated_chains, telegram_channel, cmd_handlers_rabbitmq,
cmd_handlers_redis, cmd_handlers_mongo)
handler_rabbitmq = RabbitMQApi(logger=handler_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
telegram_commands_handler = TelegramCommandsHandler(
handler_display_name, handler_logger, handler_rabbitmq,
telegram_channel, cmd_handlers)
log_and_print(
"Successfully initialised {}".format(handler_display_name),
handler_logger)
break
except Exception as e:
msg = get_initialisation_error_message(handler_display_name, e)
log_and_print(msg, handler_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return telegram_commands_handler
def __init__(self,
name: str,
logger: logging.Logger,
config_directory: str,
rabbit_ip: str,
file_patterns: Optional[List[str]] = None,
ignore_file_patterns: Optional[List[str]] = None,
ignore_directories: bool = True,
case_sensitive: bool = False):
"""
Constructs the ConfigsManager instance
:param config_directory: The root config directory to watch.
This is searched recursively.
:param file_patterns: The file patterns in the directory to watch.
Defaults to all ini files
:param ignore_file_patterns: Any file patterns to ignore.
Defaults to None
:param ignore_directories: Whether changes in directories should be
ignored. Default: True
:param case_sensitive: Whether the patterns in `file_patterns` and
`ignore_file_patterns` are case sensitive. Defaults to False
"""
if not file_patterns:
file_patterns = ['*.ini']
self._name = name
self._config_directory = config_directory
self._file_patterns = file_patterns
self._watching = False
self._connected_to_rabbit = False
logger.debug("Creating config RabbitMQ connection")
rabbitmq = RabbitMQApi(logger.getChild("config_{}".format(
RabbitMQApi.__name__)),
host=rabbit_ip)
super().__init__(logger, rabbitmq)
self._logger.debug("Creating heartbeat RabbitMQ connection")
self._heartbeat_rabbit = RabbitMQApi(logger.getChild(
"heartbeat_{}".format(RabbitMQApi.__name__)),
host=rabbit_ip)
self._event_handler = ConfigFileEventHandler(
self._logger.getChild(ConfigFileEventHandler.__name__),
self._on_event_thrown, file_patterns, ignore_file_patterns,
ignore_directories, case_sensitive)
self._observer = PollingObserver()
self._observer.schedule(self._event_handler,
config_directory,
recursive=True)
def disconnect_from_rabbit(rabbit: RabbitMQApi, attempts: int = 3) -> None:
tries = 0
while tries < attempts:
try:
rabbit.disconnect()
return
except Exception as e:
tries += 1
print("Could not disconnect to rabbit. Attempts so "
"far: {}".format(tries))
print(e)
if tries >= attempts:
raise e
def _initialise_data_transformer(data_transformer_type: Type[T],
data_transformer_display_name: str) -> T:
transformer_logger = _initialise_transformer_logger(
data_transformer_display_name, data_transformer_type.__name__)
redis = _initialise_transformer_redis(data_transformer_display_name,
transformer_logger)
# Try initialising the transformer until successful
while True:
try:
rabbitmq = RabbitMQApi(logger=transformer_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
data_transformer = data_transformer_type(
data_transformer_display_name, transformer_logger, redis,
rabbitmq, env.DATA_TRANSFORMER_PUBLISHING_QUEUE_SIZE)
log_and_print(
"Successfully initialised {}".format(
data_transformer_display_name), transformer_logger)
break
except Exception as e:
msg = get_initialisation_error_message(
data_transformer_display_name, e)
log_and_print(msg, transformer_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return data_transformer
def _initialise_log_alerts_handler(channel_id: str,
channel_name: str) -> LogAlertsHandler:
# Handler display name based on channel name
handler_display_name = LOG_ALERTS_HANDLER_NAME_TEMPLATE.format(
channel_name)
handler_logger = _initialise_channel_handler_logger(
handler_display_name, LogAlertsHandler.__name__)
alerts_logger = _initialise_alerts_logger()
# Try initialising handler until successful
while True:
try:
log_channel = LogChannel(
channel_name, channel_id,
handler_logger.getChild(LogChannel.__name__), alerts_logger)
rabbitmq = RabbitMQApi(logger=handler_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
log_alerts_handler = LogAlertsHandler(handler_display_name,
handler_logger, rabbitmq,
log_channel)
log_and_print(
"Successfully initialised {}".format(handler_display_name),
handler_logger)
break
except Exception as e:
msg = get_initialisation_error_message(handler_display_name, e)
log_and_print(msg, handler_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return log_alerts_handler
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.transformer_display_name = 'Test Data Transformer'
self.transformer_module_name = 'TestDataTransformer'
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.redis_db = env.REDIS_DB
self.redis_host = env.REDIS_IP
self.redis_port = env.REDIS_PORT
self.redis_namespace = env.UNIQUE_ALERTER_IDENTIFIER
self.redis = RedisApi(self.dummy_logger, self.redis_db,
self.redis_host, self.redis_port, '',
self.redis_namespace,
self.connection_check_time_interval)
self.github_dt_name = 'test_github_data_transformer'
self.github_dt_publishing_queue_size = 1000
self.publishing_queue_github_dt = Queue(
self.github_dt_publishing_queue_size)
self.test_github_dt = GitHubDataTransformer(
self.github_dt_name, self.dummy_logger, self.redis, self.rabbitmq,
self.github_dt_publishing_queue_size)
self.test_github_dt._publishing_queue = self.publishing_queue_github_dt
self.system_dt_name = 'test_system_data_transformer'
self.system_dt_publishing_queue_size = 1001
self.publishing_queue_system_dt = Queue(
self.system_dt_publishing_queue_size)
self.test_system_dt = SystemDataTransformer(
self.system_dt_name, self.dummy_logger, self.redis, self.rabbitmq,
self.system_dt_publishing_queue_size)
self.test_system_dt._publishing_queue = self.publishing_queue_system_dt
def _initialise_monitor(monitor_type: Type[T], monitor_display_name: str,
monitoring_period: int,
config: Union[SystemConfig, RepoConfig]) -> T:
monitor_logger = _initialise_monitor_logger(monitor_display_name,
monitor_type.__name__)
# Try initialising the monitor until successful
while True:
try:
rabbitmq = RabbitMQApi(logger=monitor_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
monitor = monitor_type(monitor_display_name, config,
monitor_logger, monitoring_period, rabbitmq)
log_and_print(
"Successfully initialised {}".format(monitor_display_name),
monitor_logger)
break
except Exception as e:
msg = get_initialisation_error_message(monitor_display_name, e)
log_and_print(msg, monitor_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return monitor
def _initialise_github_alerter_manager() -> GithubAlerterManager:
manager_display_name = GITHUB_ALERTER_MANAGER_NAME
github_alerter_manager_logger = _initialise_logger(
manager_display_name, GithubAlerterManager.__name__,
env.MANAGERS_LOG_FILE_TEMPLATE)
# Attempt to initialise the system alerters manager
while True:
try:
rabbitmq = RabbitMQApi(
logger=github_alerter_manager_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
github_alerter_manager = GithubAlerterManager(
github_alerter_manager_logger, manager_display_name, rabbitmq)
break
except Exception as e:
log_and_print(
get_initialisation_error_message(manager_display_name, e),
github_alerter_manager_logger)
log_and_print(get_reattempting_message(manager_display_name),
github_alerter_manager_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return github_alerter_manager
def _initialise_github_alerter() -> GithubAlerter:
alerter_display_name = GITHUB_ALERTER_NAME
github_alerter_logger = _initialise_alerter_logger(alerter_display_name,
GithubAlerter.__name__)
# Try initialising an alerter until successful
while True:
try:
rabbitmq = RabbitMQApi(logger=github_alerter_logger.getChild(
RabbitMQApi.__name__),
host=RABBIT_IP)
github_alerter = GithubAlerter(alerter_display_name,
github_alerter_logger, rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE)
log_and_print(
"Successfully initialised {}".format(alerter_display_name),
github_alerter_logger)
break
except Exception as e:
msg = get_initialisation_error_message(alerter_display_name, e)
log_and_print(msg, github_alerter_logger)
# sleep 10 seconds before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return github_alerter
def _initialise_channels_manager() -> ChannelsManager:
manager_display_name = CHANNELS_MANAGER_NAME
channels_manager_logger = _initialise_logger(
manager_display_name, ChannelsManager.__name__,
env.MANAGERS_LOG_FILE_TEMPLATE)
# Attempt to initialise the data transformers manager
while True:
try:
rabbitmq = RabbitMQApi(logger=channels_manager_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
channels_manager = ChannelsManager(channels_manager_logger,
manager_display_name, rabbitmq)
break
except Exception as e:
log_and_print(
get_initialisation_error_message(manager_display_name, e),
channels_manager_logger)
log_and_print(get_reattempting_message(manager_display_name),
channels_manager_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return channels_manager
def _initialise_data_store_manager() -> StoreManager:
manager_display_name = DATA_STORE_MANAGER_NAME
data_store_manager_logger = _initialise_logger(
manager_display_name, StoreManager.__name__,
env.MANAGERS_LOG_FILE_TEMPLATE)
# Attempt to initialise the data store manager
while True:
try:
rabbitmq = RabbitMQApi(logger=data_store_manager_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
data_store_manager = StoreManager(data_store_manager_logger,
manager_display_name, rabbitmq)
break
except Exception as e:
log_and_print(
get_initialisation_error_message(manager_display_name, e),
data_store_manager_logger)
log_and_print(get_reattempting_message(manager_display_name),
data_store_manager_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return data_store_manager
def _initialise_system_alerter(system_alerts_config: SystemAlertsConfig,
chain: str) -> SystemAlerter:
# Alerter display name based on system
alerter_display_name = SYSTEM_ALERTER_NAME_TEMPLATE.format(chain)
system_alerter_logger = _initialise_alerter_logger(alerter_display_name,
SystemAlerter.__name__)
# Try initialising an alerter until successful
while True:
try:
rabbitmq = RabbitMQApi(logger=system_alerter_logger.getChild(
RabbitMQApi.__name__),
host=RABBIT_IP)
system_alerter = SystemAlerter(alerter_display_name,
system_alerts_config,
system_alerter_logger, rabbitmq,
ALERTER_PUBLISHING_QUEUE_SIZE)
log_and_print(
"Successfully initialised {}".format(alerter_display_name),
system_alerter_logger)
break
except Exception as e:
msg = get_initialisation_error_message(alerter_display_name, e)
log_and_print(msg, system_alerter_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return system_alerter
def _initialise_system_monitors_manager() -> SystemMonitorsManager:
manager_display_name = SYSTEM_MONITORS_MANAGER_NAME
system_monitors_manager_logger = _initialise_logger(
manager_display_name, SystemMonitorsManager.__name__,
env.MANAGERS_LOG_FILE_TEMPLATE)
# Attempt to initialise the system monitors manager
while True:
try:
rabbit_ip = env.RABBIT_IP
rabbitmq = RabbitMQApi(
logger=system_monitors_manager_logger.getChild(
RabbitMQApi.__name__),
host=rabbit_ip)
system_monitors_manager = SystemMonitorsManager(
system_monitors_manager_logger, manager_display_name, rabbitmq)
break
except Exception as e:
log_and_print(
get_initialisation_error_message(manager_display_name, e),
system_monitors_manager_logger)
log_and_print(get_reattempting_message(manager_display_name),
system_monitors_manager_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return system_monitors_manager
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
env.RABBIT_IP,
connection_check_time_interval=self.connection_check_time_interval)
self.test_rabbit_manager = RabbitMQApi(
self.dummy_logger,
env.RABBIT_IP,
connection_check_time_interval=self.connection_check_time_interval)
self.manager_name = 'test_store_manager'
self.routing_key = 'heartbeat.manager'
self.test_queue_name = 'test queue'
self.test_store_manager = StoreManager(self.dummy_logger,
self.manager_name,
self.rabbitmq)
# Adding dummy process
self.dummy_process = Process(target=infinite_fn, args=())
self.dummy_process.daemon = True
connect_to_rabbit(self.rabbitmq)
connect_to_rabbit(self.test_rabbit_manager)
self.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, 'topic', False,
True, False, False)
self.rabbitmq.queue_declare(DATA_STORE_MAN_INPUT_QUEUE, False, True,
False, False)
self.test_rabbit_manager.queue_declare(self.test_queue_name, False,
True, False, False)
self.rabbitmq.queue_bind(DATA_STORE_MAN_INPUT_QUEUE,
HEALTH_CHECK_EXCHANGE,
DATA_STORE_MAN_INPUT_ROUTING_KEY)
self.test_data_str = 'test data'
self.test_heartbeat = {
'component_name': self.manager_name,
'is_alive': True,
'timestamp': datetime(2012, 1, 1).timestamp(),
}
self.test_exception = PANICException('test_exception', 1)
def setUp(self) -> None:
self.CONFIG_MANAGER_NAME = "Config Manager"
self.config_manager_logger = logging.getLogger("test_config_manager")
self.config_manager_logger.disabled = True
self.rabbit_logger = logging.getLogger("test_rabbit")
self.rabbit_logger.disabled = True
self.config_directory = "config"
file_patterns = ["*.ini"]
rabbit_ip = env.RABBIT_IP
self.test_config_manager = ConfigsManager(self.CONFIG_MANAGER_NAME,
self.config_manager_logger,
self.config_directory,
rabbit_ip,
file_patterns=file_patterns)
self.rabbitmq = RabbitMQApi(
self.rabbit_logger,
rabbit_ip,
connection_check_time_interval=timedelta(seconds=0))
def setUp(self) -> None:
self.test_handler_name = 'test_email_alerts_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger, self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_channel_name = 'test_email_channel'
self.test_channel_id = 'test_email1234'
self.test_channel_logger = self.dummy_logger.getChild('dummy_channel')
self.test_emails_to = ['*****@*****.**', '*****@*****.**',
'*****@*****.**']
self.test_smtp = 'test smtp server'
self.test_sender = 'test sender'
self.test_username = '******'
self.test_password = '******'
self.test_port = 10
self.test_api = EmailApi(self.test_smtp, self.test_sender,
self.test_username, self.test_password,
self.test_port)
self.test_channel = EmailChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.test_emails_to,
self.test_api)
self.test_queue_size = 1000
self.test_max_attempts = 5
self.test_alert_validity_threshold = 300
self.test_email_alerts_handler = EmailAlertsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel, self.test_queue_size,
self.test_max_attempts, self.test_alert_validity_threshold)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id
)
self.test_alerts_queue = Queue(self.test_queue_size)
def __init__(self, interval: int, logger: logging.Logger, redis: RedisApi,
name: str) -> None:
self._name = name
self._interval = interval
self._logger = logger
self._redis = redis
rabbit_ip = env.RABBIT_IP
self._rabbitmq = RabbitMQApi(
logger=self.logger.getChild(RabbitMQApi.__name__), host=rabbit_ip)
# Handle termination signals by stopping the monitor gracefully
signal.signal(signal.SIGTERM, self.on_terminate)
signal.signal(signal.SIGINT, self.on_terminate)
signal.signal(signal.SIGHUP, self.on_terminate)
def setUp(self) -> None:
self.test_handler_name = 'test_telegram_alerts_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_channel_name = 'test_telegram_channel'
self.test_channel_id = 'test_telegram_id12345'
self.test_channel_logger = self.dummy_logger.getChild('dummy_channel')
self.test_bot_token = 'test_bot_token'
self.test_bot_chat_id = 'test_bot_chat_id'
self.test_base_url = \
"https://api.telegram.org/bot" + self.test_bot_token
self.test_api = TelegramBotApi(self.test_bot_token,
self.test_bot_chat_id)
self.test_channel = TelegramChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.test_api)
self.test_queue_size = 1000
self.test_max_attempts = 5
self.test_alert_validity_threshold = 300
self.test_telegram_alerts_handler = TelegramAlertsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel, self.test_queue_size, self.test_max_attempts,
self.test_alert_validity_threshold)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id)
self.test_alerts_queue = Queue(self.test_queue_size)
def setUp(self) -> None:
self.test_handler_name = 'test_twilio_alerts_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_channel_name = 'test_twilio_channel'
self.test_channel_id = 'test_twilio_id12345'
self.test_channel_logger = self.dummy_logger.getChild('twilio_channel')
self.test_account_sid = 'test_account_sid'
self.test_auth_token = 'test_auth_token'
self.test_call_from = '+35699999999'
self.test_call_to = ['+35611111111', '+35644545454', '+35634343434']
self.test_twiml = '<Response><Reject/></Response>'
self.test_twiml_is_url = False
self.test_api = TwilioApi(self.test_account_sid, self.test_auth_token)
self.test_channel = TwilioChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.test_api)
self.test_max_attempts = 3
self.test_alert_validity_threshold = 300
self.test_twilio_alerts_handler = TwilioAlertsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel, self.test_call_from, self.test_call_to,
self.test_twiml, self.test_twiml_is_url, self.test_max_attempts,
self.test_alert_validity_threshold)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id)
def _initialise_email_alerts_handler(smtp: str,
email_from: str,
emails_to: List[str],
channel_id: str,
channel_name: str,
username: Optional[str],
password: Optional[str],
port: int = 0) -> EmailAlertsHandler:
# Handler display name based on channel name
handler_display_name = EMAIL_ALERTS_HANDLER_NAME_TEMPLATE.format(
channel_name)
handler_logger = _initialise_channel_handler_logger(
handler_display_name, EmailAlertsHandler.__name__)
# Try initialising handler until successful
while True:
try:
email_api = EmailApi(smtp, email_from, username, password, port)
email_channel = EmailChannel(
channel_name, channel_id,
handler_logger.getChild(EmailChannel.__name__), emails_to,
email_api)
rabbitmq = RabbitMQApi(logger=handler_logger.getChild(
RabbitMQApi.__name__),
host=env.RABBIT_IP)
email_alerts_handler = EmailAlertsHandler(
handler_display_name, handler_logger, rabbitmq, email_channel,
env.CHANNELS_MANAGER_PUBLISHING_QUEUE_SIZE)
log_and_print(
"Successfully initialised {}".format(handler_display_name),
handler_logger)
break
except Exception as e:
msg = get_initialisation_error_message(handler_display_name, e)
log_and_print(msg, handler_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return email_alerts_handler
def __init__(self, logger: logging.Logger, redis: RedisApi, name: str) \
-> None:
self._name = name
self._logger = logger
self._redis = redis
rabbit_ip = env.RABBIT_IP
self._rabbitmq = RabbitMQApi(logger=self.logger.getChild(
RabbitMQApi.__name__),
host=rabbit_ip)
# This dict stores the keys-values that should have been stored to redis
# but where not saved due to an error in redis. This is done so that the
# heartbeats are saved eventually redis is back online
self._unsavable_redis_data = {}
# Handle termination signals by stopping the monitor gracefully
signal.signal(signal.SIGTERM, self.on_terminate)
signal.signal(signal.SIGINT, self.on_terminate)
signal.signal(signal.SIGHUP, self.on_terminate)
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.monitor_display_name = 'Test Monitor'
self.monitor_module_name = 'TestMonitor'
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger, self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.github_monitor_name = 'test_github_monitor'
self.github_monitoring_period = env.GITHUB_MONITOR_PERIOD_SECONDS
self.github_repo_id = 'test_repo_id'
self.github_parent_id = 'test_github_parent_id'
self.github_repo_name = 'test_repo'
self.monitor_repo = True
self.releases_page = 'test_url'
self.repo_config = RepoConfig(self.github_repo_id,
self.github_parent_id,
self.github_repo_name, self.monitor_repo,
self.releases_page)
self.test_github_monitor = GitHubMonitor(self.github_monitor_name,
self.repo_config,
self.dummy_logger,
self.github_monitoring_period,
self.rabbitmq)
self.system_monitor_name = 'test_system_monitor'
self.system_monitoring_period = env.SYSTEM_MONITOR_PERIOD_SECONDS
self.system_id = 'test_system_id'
self.system_parent_id = 'test_system_parent_id'
self.system_name = 'test_system'
self.monitor_system = True
self.node_exporter_url = 'test_url'
self.system_config = SystemConfig(self.system_id, self.system_parent_id,
self.system_name, self.monitor_system,
self.node_exporter_url)
self.test_system_monitor = SystemMonitor(self.system_monitor_name,
self.system_config,
self.dummy_logger,
self.system_monitoring_period,
self.rabbitmq)
def __init__(self, name: str, logger: Logger, rabbit_ip: str,
redis_ip: str, redis_db: int, redis_port: int,
unique_alerter_identifier: str, enable_console_alerts: bool,
enable_log_alerts: bool):
self._name = name
self._redis = RedisApi(logger.getChild(RedisApi.__name__),
host=redis_ip,
db=redis_db,
port=redis_port,
namespace=unique_alerter_identifier)
self._enable_console_alerts = enable_console_alerts
self._enable_log_alerts = enable_log_alerts
self._config = {}
super().__init__(
logger,
RabbitMQApi(logger=logger.getChild(RabbitMQApi.__name__),
host=rabbit_ip),
env.ALERT_ROUTER_PUBLISHING_QUEUE_SIZE)
def _initialise_store(store_type: Type[T], store_display_name: str) -> T:
store_logger = _initialise_store_logger(store_display_name,
store_type.__name__)
# Try initialising the store until successful
while True:
try:
rabbitmq = RabbitMQApi(
logger=store_logger.getChild(RabbitMQApi.__name__),
host=env.RABBIT_IP)
store = store_type(store_display_name, store_logger, rabbitmq)
log_and_print("Successfully initialised {}".format(
store_display_name), store_logger)
break
except Exception as e:
msg = get_initialisation_error_message(store_display_name, e)
log_and_print(msg, store_logger)
# sleep before trying again
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
return store
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.store_name = 'test_store'
self.github_store_name = 'test_github_store'
self.system_store_name = 'test_system_store'
self.alerter_store_name = 'alerter_store_name'
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_github_store = GithubStore(GITHUB_STORE_NAME,
self.dummy_logger, self.rabbitmq)
self.test_system_store = SystemStore(SYSTEM_STORE_NAME,
self.dummy_logger, self.rabbitmq)
self.test_alert_store = AlertStore(ALERT_STORE_NAME, self.dummy_logger,
self.rabbitmq)
self.test_config_store = ConfigStore(CONFIG_STORE_NAME,
self.dummy_logger, self.rabbitmq)
def setUp(self) -> None:
self.test_handler_name = 'test_log_alerts_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_channel_name = 'test_logger_channel'
self.test_channel_id = 'test_logger1234'
self.test_channel_logger = self.dummy_logger.getChild('dummy_channel')
self.alerts_logger = self.dummy_logger.getChild('alerts_logger')
self.test_channel = LogChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.alerts_logger)
self.test_log_alerts_handler = LogAlertsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id)