def send_event(service, check_name, overrides, status, output, soa_dir, ttl=None):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param check_name: The name of the check as it appears in Sensu
:param overrides: A dictionary containing overrides for monitoring options
(e.g. notification_email, ticket, page)
:param status: The status to emit for this event
:param output: The output to emit for this event
:param soa_dir: The service directory to read monitoring information from
"""
# This function assumes the input is a string like "mumble.main"
team = get_team(overrides, service, soa_dir)
if not team:
return
runbook = overrides.get("runbook", "http://y/paasta-troubleshooting")
result_dict = {
"tip": get_tip(overrides, service, soa_dir),
"notification_email": get_notification_email(overrides, service, soa_dir),
"irc_channels": get_irc_channels(overrides, service, soa_dir),
"ticket": get_ticket(overrides, service, soa_dir),
"project": get_project(overrides, service, soa_dir),
"page": get_page(overrides, service, soa_dir),
"alert_after": overrides.get("alert_after", "5m"),
"check_every": overrides.get("check_every", "1m"),
"realert_every": -1,
"source": "paasta-%s" % load_system_paasta_config().get_cluster(),
"ttl": ttl,
}
pysensu_yelp.send_event(check_name, runbook, status, output, team, **result_dict)
def send_event(service,
check_name,
overrides,
status,
output,
soa_dir,
ttl=None):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param check_name: The name of the check as it appears in Sensu
:param overrides: A dictionary containing overrides for monitoring options
(e.g. notification_email, ticket, page)
:param status: The status to emit for this event
:param output: The output to emit for this event
:param soa_dir: The service directory to read monitoring information from
"""
# This function assumes the input is a string like "mumble.main"
team = get_team(overrides, service, soa_dir)
if not team:
return
runbook = overrides.get('runbook', 'http://y/paasta-troubleshooting')
system_paasta_config = load_system_paasta_config()
result_dict = {
'tip':
get_tip(overrides, service, soa_dir),
'notification_email':
get_notification_email(overrides, service, soa_dir),
'irc_channels':
get_irc_channels(overrides, service, soa_dir),
'ticket':
get_ticket(overrides, service, soa_dir),
'project':
get_project(overrides, service, soa_dir),
'page':
get_page(overrides, service, soa_dir),
'alert_after':
overrides.get('alert_after', '5m'),
'check_every':
overrides.get('check_every', '1m'),
'realert_every':
overrides.get('realert_every', monitoring_defaults('realert_every')),
'source':
'paasta-%s' % system_paasta_config.get_cluster(),
'ttl':
ttl,
}
sensu_host = system_paasta_config.get_sensu_host()
sensu_port = system_paasta_config.get_sensu_port()
if sensu_host is not None:
pysensu_yelp.send_event(check_name,
runbook,
status,
output,
team,
sensu_host=sensu_host,
sensu_port=sensu_port,
**result_dict)
def send_event(service, check_name, overrides, status, output, soa_dir, ttl=None):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param check_name: The name of the check as it appears in Sensu
:param overrides: A dictionary containing overrides for monitoring options
(e.g. notification_email, ticket, page)
:param status: The status to emit for this event
:param output: The output to emit for this event
:param soa_dir: The service directory to read monitoring information from
"""
# This function assumes the input is a string like "mumble.main"
team = get_team(overrides, service, soa_dir)
if not team:
return
runbook = overrides.get('runbook', 'http://y/paasta-troubleshooting')
result_dict = {
'tip': get_tip(overrides, service, soa_dir),
'notification_email': get_notification_email(overrides, service, soa_dir),
'irc_channels': get_irc_channels(overrides, service, soa_dir),
'ticket': get_ticket(overrides, service, soa_dir),
'project': get_project(overrides, service, soa_dir),
'page': get_page(overrides, service, soa_dir),
'alert_after': overrides.get('alert_after', '5m'),
'check_every': overrides.get('check_every', '1m'),
'realert_every': -1,
'source': 'paasta-%s' % load_system_paasta_config().get_cluster(),
'ttl': ttl,
}
pysensu_yelp.send_event(check_name, runbook, status, output, team, **result_dict)
def test_send_event_custom_sensu_host(self):
magic_skt = mock.MagicMock()
with mock.patch('socket.socket', return_value=magic_skt) as skt_patch:
pysensu_yelp.send_event(self.test_name, self.test_runbook,
self.test_status, self.test_output,
self.test_team, page=self.test_page, tip=self.test_tip,
notification_email=self.test_notification_email,
check_every=self.test_check_every,
realert_every=self.test_realert_every,
alert_after=self.test_alert_after,
dependencies=self.test_dependencies,
irc_channels=self.test_irc_channels,
slack_channels=self.test_slack_channels,
ticket=self.test_ticket,
project=self.test_project,
priority=self.test_priority,
source=self.test_source,
tags=self.test_tags,
ttl=self.test_ttl,
sensu_host='testhost',
sensu_port=666)
assert skt_patch.call_count == 1
magic_skt.connect.assert_called_once_with(('testhost', 666))
magic_skt.sendall.assert_called_once_with(self.event_hash + b'\n')
magic_skt.close.assert_called_once()
def enable(self, new_enable_value):
if self._enable and not new_enable_value:
# send final message without ttl
final_sensu_info = copy.deepcopy(self._sensu_event_info)
final_sensu_info.pop('ttl')
pysensu_yelp.send_event(**final_sensu_info)
self._enable = new_enable_value
def enable(self, new_enable_value):
if self._enable and not new_enable_value:
# send final message without ttl
final_sensu_info = copy.deepcopy(self._sensu_event_info)
final_sensu_info.pop('ttl')
pysensu_yelp.send_event(**final_sensu_info)
self._enable = new_enable_value
def send_event(
service, check_name, overrides, status, output, soa_dir, ttl=None, cluster=None
):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param check_name: The name of the check as it appears in Sensu
:param overrides: A dictionary containing overrides for monitoring options
(e.g. notification_email, ticket, page)
:param status: The status to emit for this event
:param output: The output to emit for this event
:param soa_dir: The service directory to read monitoring information from
:param cluster: The cluster name (optional)
"""
# This function assumes the input is a string like "mumble.main"
team = get_team(overrides, service, soa_dir)
if not team:
return
system_paasta_config = load_system_paasta_config()
if cluster is None:
try:
cluster = system_paasta_config.get_cluster()
except PaastaNotConfiguredError:
cluster = "localhost"
result_dict = {
"name": check_name,
"runbook": overrides.get("runbook", "http://y/paasta-troubleshooting"),
"status": status,
"output": output,
"team": team,
"page": get_page(overrides, service, soa_dir),
"tip": get_tip(overrides, service, soa_dir),
"notification_email": get_notification_email(overrides, service, soa_dir),
"check_every": overrides.get("check_every", "1m"),
"realert_every": overrides.get(
"realert_every", monitoring_defaults("realert_every")
),
"alert_after": overrides.get("alert_after", "5m"),
"irc_channels": get_irc_channels(overrides, service, soa_dir),
"slack_channels": get_slack_channels(overrides, service, soa_dir),
"ticket": get_ticket(overrides, service, soa_dir),
"project": get_project(overrides, service, soa_dir),
"priority": get_priority(overrides, service, soa_dir),
"source": "paasta-%s" % cluster,
"tags": get_tags(overrides, service, soa_dir),
"ttl": ttl,
"sensu_host": system_paasta_config.get_sensu_host(),
"sensu_port": system_paasta_config.get_sensu_port(),
"component": get_component(overrides, service, soa_dir),
"description": get_description(overrides, service, soa_dir),
}
if result_dict.get("sensu_host"):
pysensu_yelp.send_event(**result_dict)
def check_job_result(job, client, dry_run):
result = check_job(job, client)
if result is None:
return
if dry_run:
log.info("Would have sent this event to sensu: ")
log.info(result)
else:
log.debug("Sending event: {}".format(result))
if 'runbook' not in result:
result[
'runbook'] = "No runbook specified. Please specify a runbook in the monitoring section of the job definition."
send_event(**result)
def send_event(status, output):
result_dict = {
"name": "cluster_utilization",
"team": "paasta",
"runbook": "y/rb-mesos",
"tip": "See the mesos web interface?",
"page": True,
"alert_after": "15m",
"check_every": "5m",
"realert_every": -1,
"status": status,
"output": output,
"source": "paasta-%s" % load_system_paasta_config().get_cluster(),
}
pysensu_yelp.send_event(**result_dict)
def check_job_result(job, client, url_index, dry_run):
results = check_job(job, client, url_index)
if not results:
return
for result in results:
if dry_run:
log.info("Would have sent this event to sensu: ")
log.info(pprint.pformat(result))
else:
log.debug(f"Sending event: {pprint.pformat(result)}")
if 'runbook' not in result:
result[
'runbook'] = "No runbook specified. Please specify a runbook in the monitoring section of the job definition.",
send_event(**result)
def send_event(status, output):
result_dict = {
'name': 'cluster_utilization',
'team': 'paasta',
'runbook': 'y/rb-mesos',
'tip': 'See the mesos web interface?',
'page': True,
'alert_after': '15m',
'check_every': '5m',
'realert_every': -1,
'status': status,
'output': output,
'source': 'paasta-%s' % load_system_paasta_config().get_cluster()
}
pysensu_yelp.send_event(**result_dict)
def send_event(status, output):
result_dict = {
'name': 'cluster_utilization',
'team': 'paasta',
'runbook': 'y/rb-mesos',
'tip': 'See the mesos web interface?',
'page': True,
'alert_after': '15m',
'check_every': '5m',
'realert_every': -1,
'status': status,
'output': output,
'source': 'paasta-%s' % load_system_paasta_config().get_cluster()
}
pysensu_yelp.send_event(**result_dict)
def check_job_result(job, client, dry_run):
results = check_job(job, client)
if not results:
return
for result in results:
if dry_run:
log.info("Would have sent this event to sensu: ")
log.info(pprint.pformat(result))
else:
log.debug("Sending event: {}".format(pprint.pformat(result)))
if 'runbook' not in result:
result[
'runbook'
] = "No runbook specified. Please specify a runbook in the monitoring section of the job definition.",
send_event(**result)
def test_send_event_no_team(self):
magic_skt = mock.MagicMock()
with mock.patch('socket.socket', return_value=magic_skt) as skt_patch:
with pytest.raises(ValueError):
pysensu_yelp.send_event(self.test_name, self.test_runbook,
self.test_status, self.test_output,
'', page=self.test_page, tip=self.test_tip,
notification_email=self.test_notification_email,
check_every=self.test_check_every,
realert_every=self.test_realert_every,
alert_after=self.test_alert_after,
dependencies=self.test_dependencies,
irc_channels=self.test_irc_channels,
ticket=self.test_ticket,
project=self.test_project,
source=self.test_source,
ttl=self.test_ttl)
skt_patch.assert_not_called()
def test_send_event_no_team(self):
magic_skt = mock.MagicMock()
with mock.patch('socket.socket', return_value=magic_skt) as skt_patch:
with pytest.raises(ValueError):
pysensu_yelp.send_event(self.test_name, self.test_runbook,
self.test_status, self.test_output,
'', page=self.test_page, tip=self.test_tip,
notification_email=self.test_notification_email,
check_every=self.test_check_every,
realert_every=self.test_realert_every,
alert_after=self.test_alert_after,
dependencies=self.test_dependencies,
irc_channels=self.test_irc_channels,
ticket=self.test_ticket,
project=self.test_project,
source=self.test_source,
watchdog_timer=self.test_watchdog_timer)
skt_patch.assert_not_called()
def alert(self, msg: str):
import pysensu_yelp
result_dict = {
'name': 'tron_dynamodb_check',
'runbook': '',
'status': 1,
'output': msg,
'team': 'compute-infra',
'tip': '',
'page': None,
'notification_email': '*****@*****.**',
'irc_channels': None,
'slack_channels': None,
'alert_after': '1m',
'check_every': '10m',
'realert_every': -1,
'ttl': None
}
pysensu_yelp.send_event(**result_dict)
def test_no_special_characters_in_name(self):
magic_skt = mock.MagicMock()
with mock.patch('socket.socket', return_value=magic_skt):
for special_char in '!@#$%^&*() ;",<>=+[]':
test_name = self.test_name + special_char
with pytest.raises(ValueError):
pysensu_yelp.send_event(test_name, self.test_runbook,
self.test_status, self.test_output,
self.test_team, page=self.test_page, tip=self.test_tip,
notification_email=self.test_notification_email,
check_every=self.test_check_every,
realert_every=self.test_realert_every,
alert_after=self.test_alert_after,
dependencies=self.test_dependencies,
irc_channels=self.test_irc_channels,
ticket=self.test_ticket,
project=self.test_project,
source=self.test_source,
ttl=self.test_ttl)
def alert(self, name: str, msg: str, error: str):
import pysensu_yelp
result_dict = {
'name': name,
'runbook': '',
'status': 1,
'output': '\n'.join(msg, error),
'team': 'compute-infra',
'tip': '',
'page': None,
'notification_email': '*****@*****.**',
'irc_channels': None,
'slack_channels': None,
'alert_after': '1m',
'check_every': '10m',
'realert_every': -1,
'ttl': None
}
pysensu_yelp.send_event(**result_dict)
def test_no_special_characters_in_name(self):
magic_skt = mock.MagicMock()
with mock.patch('socket.socket', return_value=magic_skt) as skt_patch:
for special_char in '!@#$%^&*() ;",<>=+[]':
test_name = self.test_name + special_char
with pytest.raises(ValueError):
pysensu_yelp.send_event(test_name, self.test_runbook,
self.test_status, self.test_output,
self.test_team, page=self.test_page, tip=self.test_tip,
notification_email=self.test_notification_email,
check_every=self.test_check_every,
realert_every=self.test_realert_every,
alert_after=self.test_alert_after,
dependencies=self.test_dependencies,
irc_channels=self.test_irc_channels,
ticket=self.test_ticket,
project=self.test_project,
source=self.test_source,
watchdog_timer=self.test_watchdog_timer)
def test_send_event_valid_args(self):
magic_skt = mock.MagicMock()
with mock.patch('socket.socket', return_value=magic_skt) as skt_patch:
pysensu_yelp.send_event(self.test_name, self.test_runbook,
self.test_status, self.test_output,
self.test_team, page=self.test_page, tip=self.test_tip,
notification_email=self.test_notification_email,
check_every=self.test_check_every,
realert_every=self.test_realert_every,
alert_after=self.test_alert_after,
dependencies=self.test_dependencies,
irc_channels=self.test_irc_channels,
ticket=self.test_ticket,
project=self.test_project,
source=self.test_source,
ttl=self.test_ttl)
assert skt_patch.call_count == 1
magic_skt.connect.assert_called_once_with(('localhost', 3030))
magic_skt.sendall.assert_called_once_with(self.event_hash + '\n')
magic_skt.close.assert_called_once()
def test_send_event_valid_args(self):
magic_skt = mock.MagicMock()
with mock.patch('socket.socket', return_value=magic_skt) as skt_patch:
pysensu_yelp.send_event(self.test_name, self.test_runbook,
self.test_status, self.test_output,
self.test_team, page=self.test_page, tip=self.test_tip,
notification_email=self.test_notification_email,
check_every=self.test_check_every,
realert_every=self.test_realert_every,
alert_after=self.test_alert_after,
dependencies=self.test_dependencies,
irc_channels=self.test_irc_channels,
ticket=self.test_ticket,
project=self.test_project,
source=self.test_source,
watchdog_timer=self.test_watchdog_timer)
skt_patch.assert_called_once()
magic_skt.connect.assert_called_once_with(pysensu_yelp.SENSU_ON_LOCALHOST)
magic_skt.sendall.assert_called_once_with(self.event_hash + '\n')
magic_skt.close.assert_called_once()
def process(self):
if self.enable:
pysensu_yelp.send_event(**self._sensu_event_info)
def report_event(event_dict):
assert event_dict['team'] is not None
pysensu_yelp.send_event(**event_dict)
def send_event(
service,
check_name,
overrides,
status,
output,
soa_dir,
ttl=None,
cluster=None,
system_paasta_config=None,
dry_run=False,
):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param check_name: The name of the check as it appears in Sensu
:param overrides: A dictionary containing overrides for monitoring options
(e.g. notification_email, ticket, page)
:param status: The status to emit for this event
:param output: The output to emit for this event
:param soa_dir: The service directory to read monitoring information from
:param ttl: TTL (optional)
:param cluster: The cluster name (optional)
:param system_paasta_config: A SystemPaastaConfig object representing the system
:param dry_run: Print the Sensu event instead of emitting it
"""
# This function assumes the input is a string like "mumble.main"
team = get_team(overrides, service, soa_dir)
if not team:
return
if system_paasta_config is None:
system_paasta_config = load_system_paasta_config()
if cluster is None:
try:
cluster = system_paasta_config.get_cluster()
except PaastaNotConfiguredError:
cluster = "localhost"
alert_after = overrides.get("alert_after", "5m")
result_dict = {
"name":
check_name,
"runbook":
overrides.get("runbook", "http://y/paasta-troubleshooting"),
"status":
status,
"output":
output,
"team":
team,
"page":
get_page(overrides, service, soa_dir),
"tip":
get_tip(overrides, service, soa_dir),
"notification_email":
get_notification_email(overrides, service, soa_dir),
"check_every":
overrides.get("check_every", "1m"),
"realert_every":
overrides.get("realert_every", monitoring_defaults("realert_every")),
"alert_after":
f"{alert_after}s" if isinstance(alert_after, int) else alert_after,
"irc_channels":
get_irc_channels(overrides, service, soa_dir),
"slack_channels":
get_slack_channels(overrides, service, soa_dir),
"ticket":
get_ticket(overrides, service, soa_dir),
"project":
get_project(overrides, service, soa_dir),
"priority":
get_priority(overrides, service, soa_dir),
"source":
"paasta-%s" % cluster,
"tags":
get_tags(overrides, service, soa_dir),
"ttl":
ttl,
"sensu_host":
system_paasta_config.get_sensu_host(),
"sensu_port":
system_paasta_config.get_sensu_port(),
"component":
get_component(overrides, service, soa_dir),
"description":
get_description(overrides, service, soa_dir),
}
if dry_run:
if status == pysensu_yelp.Status.OK:
print(f"Would've sent an OK event for check '{check_name}'")
else:
from pprint import pprint # only import during testing
print(
f"Would've sent the following alert for check '{check_name}':")
pprint(result_dict)
elif result_dict.get("sensu_host"):
pysensu_yelp.send_event(**result_dict)
def alert(self, repo_name, secrets):
self.config_data['output'] = "In repo " + repo_name + "\n" + str(
secrets)
pysensu_yelp.send_event(**self.config_data)
def report_event(event_dict):
assert event_dict['team'] is not None
pysensu_yelp.send_event(**event_dict)
def process(self):
if self.enable:
pysensu_yelp.send_event(**self._sensu_event_info)
def sensu_checkin(
*,
check_name: str,
output: str,
source: str,
status: Status = Status.OK,
app: Optional[str] = None,
pool: Optional[str] = None,
scheduler: Optional[str] = None,
noop: bool = False,
page: bool = True,
**kwargs: Any,
) -> None:
# This function feels like a massive hack, let's revisit and see if we can make it better (CLUSTERMAN-304)
#
# TODO (CLUSTERMAN-126) right now there's only one app per pool so use the global pool namespace
# We assume the "pool" name and the "app" name are the same
#
# Use 'no-namespace' instead of None so we don't skip the per-cluster override
pool_namespace = POOL_NAMESPACE.format(
pool=app, scheduler=scheduler) if app else 'no-namespace'
# read the sensu configuration from srv-configs; signals are not required to define this, so in the case
# that they do not define anything, we fall back to the clusterman config. The clusterman config can override
# alerts on a per-cluster basis, so first check there; if nothing is defined there, fall back to the default,
# which is required to be defined, so we know that someone is going to get the notification
#
sensu_config = dict(
staticconf.read_list('sensu_config',
default=[{}],
namespace=pool_namespace).pop())
if not sensu_config:
sensu_config = dict(
staticconf.read_list(f'clusters.{source}.sensu_config',
default=[{}]).pop())
if not sensu_config:
sensu_config = dict(staticconf.read_list('sensu_config').pop())
# If we've turned off paging in the config, we don't want this function to ever page
config_page = sensu_config.pop('page', None)
page = False if config_page is False else page
# So we know where alerts are coming from precisely
output += ''.join([
'\n\nThis check came from:\n',
f'- Cluster/region: {source}\n',
f'- Pool: {pool}.{scheduler}\n' if pool else '',
f'- App: {app}\n' if app else '',
])
sensu_config.update({
'name': check_name,
'output': output,
'source': source,
'status': status.value,
'page': page,
})
# values passed in to this function override config file values (is this really correct??)
sensu_config.update(kwargs)
pysensu_yelp = _get_sensu()
if noop or not pysensu_yelp:
logger.info(('Would have sent this event to Sensu:\n'
f'{pprint.pformat(sensu_config)}'))
return
# team and runbook are required entries in srv-configs, so we know this will go to the "right" place
pysensu_yelp.send_event(**sensu_config)
def log_and_send_event(self, output_dict):
self._log.info("{} status: {}, output: {}.".format(
self._service_name, output_dict['status'], output_dict['output']))
pysensu_yelp.send_event(**output_dict)
def send_event(service,
check_name,
overrides,
status,
output,
soa_dir,
ttl=None,
cluster=None):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param check_name: The name of the check as it appears in Sensu
:param overrides: A dictionary containing overrides for monitoring options
(e.g. notification_email, ticket, page)
:param status: The status to emit for this event
:param output: The output to emit for this event
:param soa_dir: The service directory to read monitoring information from
:param cluster: The cluster name (optional)
"""
# This function assumes the input is a string like "mumble.main"
team = get_team(overrides, service, soa_dir)
if not team:
return
system_paasta_config = load_system_paasta_config()
if cluster is None:
try:
cluster = system_paasta_config.get_cluster()
except PaastaNotConfiguredError:
cluster = 'localhost'
result_dict = {
'name':
check_name,
'runbook':
overrides.get('runbook', 'http://y/paasta-troubleshooting'),
'status':
status,
'output':
output,
'team':
team,
'page':
get_page(overrides, service, soa_dir),
'tip':
get_tip(overrides, service, soa_dir),
'notification_email':
get_notification_email(overrides, service, soa_dir),
'check_every':
overrides.get('check_every', '1m'),
'realert_every':
overrides.get('realert_every', monitoring_defaults('realert_every')),
'alert_after':
overrides.get('alert_after', '5m'),
'irc_channels':
get_irc_channels(overrides, service, soa_dir),
'slack_channels':
get_slack_channels(overrides, service, soa_dir),
'ticket':
get_ticket(overrides, service, soa_dir),
'project':
get_project(overrides, service, soa_dir),
'priority':
get_priority(overrides, service, soa_dir),
'source':
'paasta-%s' % cluster,
'tags':
get_tags(overrides, service, soa_dir),
'ttl':
ttl,
'sensu_host':
system_paasta_config.get_sensu_host(),
'sensu_port':
system_paasta_config.get_sensu_port(),
'component':
get_component(overrides, service, soa_dir),
'description':
get_description(overrides, service, soa_dir),
}
if result_dict.get('sensu_host'):
pysensu_yelp.send_event(**result_dict)
