def test_get_runbook(self):
with mock.patch(
'paasta_tools.monitoring_tools.__get_monitoring_config_value',
) as get_monitoring_config_value_patch:
monitoring_tools.get_runbook(self.overrides, self.service, self.soa_dir)
get_monitoring_config_value_patch.assert_called_once_with('runbook', self.overrides, self.service,
self.soa_dir)
def test_get_runbook(self):
with mock.patch(
'paasta_tools.monitoring_tools.__get_monitoring_config_value',
) as get_monitoring_config_value_patch:
monitoring_tools.get_runbook(self.overrides, self.service, self.soa_dir)
get_monitoring_config_value_patch.assert_called_once_with('runbook', self.overrides, self.service,
self.soa_dir)
def test_get_runbook(self):
with mock.patch(
"paasta_tools.monitoring_tools.__get_monitoring_config_value", autospec=True
) as get_monitoring_config_value_patch:
monitoring_tools.get_runbook(self.overrides, self.service, self.soa_dir)
get_monitoring_config_value_patch.assert_called_once_with(
"runbook", self.overrides, self.service, self.soa_dir
)
def send_event(service, namespace, cluster, soa_dir, status, output):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param namespace: The namespace of the service the event is about
:param soa_dir: The service directory to read monitoring information from
:param status: The status to emit for this event
:param output: The output to emit for this event"""
# This function assumes the input is a string like "mumble.main"
monitoring_overrides = marathon_tools.load_marathon_service_config(service, namespace, cluster).get_monitoring()
if "alert_after" not in monitoring_overrides:
monitoring_overrides["alert_after"] = "2m"
monitoring_overrides["check_every"] = "1m"
monitoring_overrides["runbook"] = monitoring_tools.get_runbook(monitoring_overrides, service, soa_dir=soa_dir)
check_name = "check_marathon_services_replication.%s" % compose_job_id(service, namespace)
monitoring_tools.send_event(service, check_name, monitoring_overrides, status, output, soa_dir)
_log(
service=service,
line="Replication: %s" % output,
component="monitoring",
level="debug",
cluster=cluster,
instance=namespace,
)
def compose_monitoring_overrides_for_service(chronos_job_config):
""" Compose a group of monitoring overrides """
monitoring_overrides = chronos_job_config.get_monitoring()
if 'alert_after' not in monitoring_overrides:
monitoring_overrides['alert_after'] = '15m'
monitoring_overrides['check_every'] = '1m'
monitoring_overrides['runbook'] = monitoring_tools.get_runbook(
monitoring_overrides,
chronos_job_config.service,
soa_dir=chronos_job_config.soa_dir,
)
if 'realert_every' not in monitoring_overrides:
guessed_realert_every = guess_realert_every(chronos_job_config)
if guessed_realert_every is not None:
monitoring_overrides[
'realert_every'] = monitoring_tools.get_realert_every(
monitoring_overrides,
chronos_job_config.service,
soa_dir=chronos_job_config.soa_dir,
monitoring_defaults=lambda x:
{'realert_every': guessed_realert_every}.get(x),
)
else:
monitoring_overrides[
'realert_every'] = monitoring_tools.get_realert_every(
monitoring_overrides,
chronos_job_config.service,
soa_dir=chronos_job_config.soa_dir,
)
return monitoring_overrides
def send_event(service, namespace, cluster, soa_dir, status, output):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param namespace: The namespace of the service the event is about
:param soa_dir: The service directory to read monitoring information from
:param status: The status to emit for this event
:param output: The output to emit for this event"""
# This function assumes the input is a string like "mumble.main"
monitoring_overrides = marathon_tools.load_marathon_service_config(
service, namespace, cluster).get_monitoring()
if 'alert_after' not in monitoring_overrides:
monitoring_overrides['alert_after'] = '2m'
monitoring_overrides['check_every'] = '1m'
monitoring_overrides['runbook'] = monitoring_tools.get_runbook(monitoring_overrides, service, soa_dir=soa_dir)
check_name = 'check_marathon_services_replication.%s' % compose_job_id(service, namespace)
monitoring_tools.send_event(service, check_name, monitoring_overrides, status, output, soa_dir)
_log(
service=service,
line='Replication: %s' % output,
component='monitoring',
level='debug',
cluster=cluster,
instance=namespace,
)
def get_service_info(service, soa_dir):
service_configuration = read_service_configuration(service, soa_dir)
description = service_configuration.get('description',
NO_DESCRIPTION_MESSAGE)
external_link = service_configuration.get('external_link',
NO_EXTERNAL_LINK_MESSAGE)
smartstack_endpoints = get_smartstack_endpoints(service, soa_dir)
git_url = get_git_url(service, soa_dir)
output = []
output.append('Service Name: %s' % service)
output.append('Description: %s' % description)
output.append('External Link: %s' % PaastaColors.cyan(external_link))
output.append('Monitored By: team %s' %
get_team(service=service, overrides={}))
output.append(
'Runbook: %s' %
PaastaColors.cyan(get_runbook(service=service, overrides={})))
output.append('Git Repo: %s' % git_url)
output.append('Deployed to the following clusters:')
output.extend(get_deployments_strings(service, soa_dir))
if smartstack_endpoints:
output.append('Smartstack endpoint(s):')
for endpoint in smartstack_endpoints:
output.append(' - %s' % endpoint)
output.append('Dashboard(s):')
output.extend(get_dashboard_urls(service))
return '\n'.join(output)
def get_service_info(service):
service_configuration = read_service_configuration(service)
description = service_configuration.get('description', NO_DESCRIPTION_MESSAGE)
external_link = service_configuration.get('external_link', NO_EXTERNAL_LINK_MESSAGE)
pipeline_url = get_pipeline_url(service)
smartstack_endpoints = get_smartstack_endpoints(service)
git_url = get_git_url(service)
output = []
output.append('Service Name: %s' % service)
output.append('Description: %s' % description)
output.append('External Link: %s' % PaastaColors.cyan(external_link))
output.append('Monitored By: team %s' % get_team(service=service, overrides={}))
output.append('Runbook: %s' % PaastaColors.cyan(get_runbook(service=service, overrides={})))
output.append('Git Repo: %s' % git_url)
output.append('Jenkins Pipeline: %s' % pipeline_url)
output.append('Deployed to the following clusters:')
output.extend(get_deployments_strings(service))
if smartstack_endpoints:
output.append('Smartstack endpoint(s):')
for endpoint in smartstack_endpoints:
output.append(' - %s' % endpoint)
output.append('Dashboard(s):')
output.extend(get_dashboard_urls(service))
return '\n'.join(output)
def get_service_info(service, soa_dir):
service_configuration = read_service_configuration(service, soa_dir)
description = service_configuration.get("description",
NO_DESCRIPTION_MESSAGE)
external_link = service_configuration.get("external_link",
NO_EXTERNAL_LINK_MESSAGE)
smartstack_endpoints = get_smartstack_endpoints(service, soa_dir)
git_url = get_git_url(service, soa_dir)
output = []
output.append("Service Name: %s" % service)
output.append("Description: %s" % description)
output.append("External Link: %s" % PaastaColors.cyan(external_link))
output.append("Monitored By: team %s" %
get_team(service=service, overrides={}, soa_dir=soa_dir))
output.append("Runbook: %s" % PaastaColors.cyan(
get_runbook(service=service, overrides={}, soa_dir=soa_dir)))
output.append("Git Repo: %s" % git_url)
output.append("Deployed to the following clusters:")
output.extend(get_deployments_strings(service, soa_dir))
if smartstack_endpoints:
output.append("Smartstack endpoint(s):")
for endpoint in smartstack_endpoints:
output.append(" - %s" % endpoint)
output.append("Dashboard(s):")
output.extend(get_dashboard_urls(service))
return "\n".join(output)
def compose_monitoring_overrides_for_service(chronos_job_config):
""" Compose a group of monitoring overrides """
monitoring_overrides = chronos_job_config.get_monitoring()
if "alert_after" not in monitoring_overrides:
monitoring_overrides["alert_after"] = "15m"
monitoring_overrides["check_every"] = "1m"
monitoring_overrides["runbook"] = monitoring_tools.get_runbook(
monitoring_overrides,
chronos_job_config.service,
soa_dir=chronos_job_config.soa_dir,
)
if "realert_every" not in monitoring_overrides:
guessed_realert_every = guess_realert_every(chronos_job_config)
if guessed_realert_every is not None:
monitoring_overrides[
"realert_every"] = monitoring_tools.get_realert_every(
monitoring_overrides,
chronos_job_config.service,
soa_dir=chronos_job_config.soa_dir,
monitoring_defaults=lambda x:
{"realert_every": guessed_realert_every}.get(x),
)
else:
monitoring_overrides[
"realert_every"] = monitoring_tools.get_realert_every(
monitoring_overrides,
chronos_job_config.service,
soa_dir=chronos_job_config.soa_dir,
)
return monitoring_overrides
def send_event(service, namespace, cluster, soa_dir, status, output):
"""Send an event to sensu via pysensu_yelp with the given information.
:param service: The service name the event is about
:param namespace: The namespace of the service the event is about
:param soa_dir: The service directory to read monitoring information from
:param status: The status to emit for this event
:param output: The output to emit for this event"""
# This function assumes the input is a string like "mumble.main"
monitoring_overrides = marathon_tools.load_marathon_service_config(
service=service,
instance=namespace,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
).get_monitoring()
if 'alert_after' not in monitoring_overrides:
monitoring_overrides['alert_after'] = '2m'
monitoring_overrides['check_every'] = '1m'
monitoring_overrides['runbook'] = monitoring_tools.get_runbook(monitoring_overrides, service, soa_dir=soa_dir)
check_name = 'check_marathon_services_replication.%s' % compose_job_id(service, namespace)
monitoring_tools.send_event(service, check_name, monitoring_overrides, status, output, soa_dir)
_log(
service=service,
line='Replication: %s' % output,
component='monitoring',
level='debug',
cluster=cluster,
instance=namespace,
)
def compose_monitoring_overrides_for_service(chronos_job_config, soa_dir):
""" Compose a group of monitoring overrides """
monitoring_overrides = chronos_job_config.get_monitoring()
if 'alert_after' not in monitoring_overrides:
monitoring_overrides['alert_after'] = '2m'
monitoring_overrides['check_every'] = '1m'
monitoring_overrides['runbook'] = monitoring_tools.get_runbook(
monitoring_overrides, chronos_job_config.service, soa_dir=soa_dir)
return monitoring_overrides
def compose_monitoring_overrides_for_service(chronos_job_config, soa_dir):
""" Compose a group of monitoring overrides """
monitoring_overrides = chronos_job_config.get_monitoring()
if 'alert_after' not in monitoring_overrides:
monitoring_overrides['alert_after'] = '2m'
monitoring_overrides['check_every'] = '1m'
monitoring_overrides['runbook'] = monitoring_tools.get_runbook(
monitoring_overrides, chronos_job_config.service, soa_dir=soa_dir)
return monitoring_overrides
def compose_monitoring_overrides_for_service(cluster, service, instance,
soa_dir):
""" Compose a group of monitoring overrides """
monitoring_overrides = chronos_tools.load_chronos_job_config(
service=service, instance=instance, cluster=cluster,
soa_dir=soa_dir).get_monitoring()
monitoring_overrides['alert_after'] = '2m'
monitoring_overrides['check_every'] = '1m'
monitoring_overrides['runbook'] = monitoring_tools.get_runbook(
monitoring_overrides, service, soa_dir=soa_dir)
return monitoring_overrides
def send_event(instance_config, status, output):
"""Send an event to sensu via pysensu_yelp with the given information.
:param instance_config: an instance of MarathonServiceConfig
:param status: The status to emit for this event
:param output: The output to emit for this event"""
# This function assumes the input is a string like "mumble.main"
monitoring_overrides = instance_config.get_monitoring()
if 'alert_after' not in monitoring_overrides:
monitoring_overrides['alert_after'] = '2m'
monitoring_overrides['check_every'] = '1m'
monitoring_overrides['runbook'] = monitoring_tools.get_runbook(
monitoring_overrides,
instance_config.service, soa_dir=instance_config.soa_dir,
)
check_name = (
'check_marathon_services_replication.%s' %
instance_config.job_id
)
monitoring_tools.send_event(
service=instance_config.service,
check_name=check_name,
overrides=monitoring_overrides,
status=status,
output=output,
soa_dir=instance_config.soa_dir,
cluster=instance_config.cluster,
)
_log(
service=instance_config.service,
line='Replication: %s' % output,
component='monitoring',
level='debug',
cluster=instance_config.cluster,
instance=instance_config.instance,
)
