def test_generate_id_different_valid_different_id():
pub = PagerDutyPublisher("url", "token")
v = Validation("low", priority=Priority.CRITICAL)
v2 = Validation("what", priority=Priority.CRITICAL)
failure = Failure("bar", v, "unable to transmogrify")
another = Failure("foo", v2, "to transmogrify")
assert pub._generate_id(failure) != pub._generate_id(another)
def test_reporter_correctly_batches(env):
reporter = env["reporter"]
publishers = []
for i in xrange(10):
publishers.append(MockPublisher())
reporter.publishers = publishers
reporter.collect(Failure("failed", Validation("valid"), "why it failed"))
reporter.collect(Failure("failed2", Validation("valid"), "why it failed"))
reporter.collect(Success("success", Validation("valid")))
reporter.report()
for pub in publishers:
assert pub.failures == 2
assert pub.successes == 1
def _run_validations(validations, reporter, processes=1, timeout=60, timeout_retries=3):
""" Run the given validations and publish the results
Sort validations by order and then run them. All results are logged
to the given reporter. Once everything has been run, the reporter
will publish.
:param validations: List of :py:class:`~.validation.Validation` objects
that Alarmageddon will perform.
:param publishers: :py:class:`~.reporter.Reporter` object that will
collect validation results and then report those results to its
publishers.
:processes: The number of worker processes to spawn. Does not run
spawn additional processes if set to 1.
:timeout: If a validation runs for longer than this number of seconds,
Alarmageddon will kill the process running it.
"""
order_dict = collections.defaultdict(list)
for validation in validations:
order_dict[validation.order].append(validation)
ordered_validations = [l for _, l in sorted(order_dict.items())]
group_failures = {}
for validation in validations:
if (validation.group is not None and
validation.group not in group_failures):
group_failures[validation.group] = []
manager = multiprocessing.Manager()
for order_set in ordered_validations:
immutable_group_failures = dict(group_failures)
results = manager.list()
for valid in order_set:
for i in xrange(timeout_retries):
#TODO: parallelize
p = multiprocessing.Process(target=_perform, args=(valid, immutable_group_failures, results))
p.start()
p.join(timeout)
if p.is_alive():
#job is taking too long, kill it
#this is messy, but we assume that if something hit the
#general alarmageddon timeout, then it's stuck somewhere
#and we can't stop it nicely
p.terminate()
logger.warn("Validation {} ran for longer than {}".format(valid, timeout))
else:
break
else:
results.append(Failure(valid.name, valid,
"{} failed to terminate (ran for {}s)".format(valid,timeout),
time=timeout))
for result in results:
if result.is_failure() and result.validation.group is not None:
group_failures[result.validation.group].append(result.description())
reporter.collect(result)
reporter.report()
def test_configure_replacement_context_email_type_found(tmpdir, smtpserver,
httpserver):
email_pub = create_default_email_publisher(tmpdir, smtpserver)
http_validator = \
HttpValidation.get(httpserver.url).expect_status_codes([200])
email_settings = {"email_type": "test_alert",
"subject": "subject test line",
"body": "body test",
"sender": {"real_name": "Alarmageddon Monitor",
"address": "*****@*****.**"},
"recipients": [{"real_name": "Test Recipient",
"address":
"*****@*****.**"}]
}
emailer.enrich(http_validator, email_settings)
result = Failure("validation name", http_validator,
description="A failure occurred.")
email_pub.configure_replacement_context(result)
replacement_context = email_pub._replacement_context
assert replacement_context["test_name"] == "validation name"
assert replacement_context["test_description"] == "A failure occurred."
assert replacement_context["env"] == "test"
assert replacement_context["email_type"] == "test_alert"
assert replacement_context["email_custom_message"] == \
"Validation failed in environment {{env}}: {{test_name}}."
def test_generate_id_consistency_ssh(tmpdir):
pub = PagerDutyPublisher("url", "token")
ssh_ctx = ssh.SshContext("ubuntu", ssh_key_file(tmpdir))
ssh_ctx2 = ssh.SshContext("ubuntu", ssh_key_file(tmpdir))
v = ssh.SshCommandValidation(ssh_ctx, "name", "cmd", hosts=["a fake host"])
v2 = ssh.SshCommandValidation(ssh_ctx2,
"name",
"cmd",
hosts=["a fake host"])
failure = Failure("bar", v, "unable to transmogrify")
another = Failure("foo", v2, "to transmogrify")
assert pub._generate_id(failure) == pub._generate_id(another)
def test_send(tmpdir, smtpserver, httpserver):
email_pub = create_default_email_publisher(tmpdir, smtpserver)
create_subject_template(tmpdir)
create_body_template(tmpdir)
http_validator = \
HttpValidation.get(httpserver.url).expect_status_codes([200])
general_defaults = email_pub._config["email_defaults"]["general"]
email_settings = {"email_type": "test_alert",
"subject": general_defaults["email_subject_template"],
"body": general_defaults["email_template"],
"sender": general_defaults["email_sender"],
"recipients": general_defaults["email_recipients"]}
emailer.enrich(http_validator, email_settings)
failure_message = "Validation failure. Expected 200, received 404."
result = Failure("Check Status Route", http_validator,
description=failure_message)
email_pub.send(result)
print(smtpserver.outbox[0])
assert len(smtpserver.outbox) == 1
payload = str(smtpserver.outbox[0].get_payload()[0])
assert payload.split('\n')[5] == "Validation Failure in environment test:"
assert payload.split('\n')[6] == "Test Name: [Check Status Route]"
assert payload.split('\n')[7] == \
"Test Description: [Validation failure. Expected 200, received 404.]"
custom_message = "Custom Message: [Validation failed in environment " \
"test: Check Status Route.]"
assert payload.split('\n')[8] == custom_message
def testSendFailure():
graphite = new_publisher()
v = Validation("low", priority=Priority.LOW)
failure = Failure("foo", v, "unable to frobnicate")
graphite.send(failure)
assert graphite._graphite.counter["passed"] == 0
assert graphite._graphite.counter["failed"] == 1
def test_message_length_capped(httpserver):
httpserver.serve_content(code=300,
headers={"content-type": "text/plain"},
content='{"mode":"NORMAL"}')
pub = PagerDutyPublisher(httpserver.url, "token")
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "-" * 2000)
message = pub._construct_message(failure)
assert len(message) == pagerduty.MAX_LEN
def test_publish_failure(httpserver):
httpserver.serve_content(code=300,
headers={"content-type": "text/plain"},
content='{"mode":"NORMAL"}')
pub = PagerDutyPublisher(httpserver.url, "token")
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "message")
with pytest.raises(PublishFailure):
pub.send(failure)
def test_publish_retries(ratelimited):
global cutoff
cutoff = 3
global hits
hits = 0
pub = PagerDutyPublisher(ratelimited.url, "token")
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "message")
pub.send(failure)
def test_construct_tree_batch():
pub = JUnitPublisher("should_not_be_created.xml")
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "message", time=30)
success = Success("foo", v, time=20)
tree = pub._construct_tree([failure, failure, success]).getroot()
assert tree.get("failures") == str(2)
assert tree.get("tests") == str(3)
assert float(tree.get("time")) == 80
assert len(tree) == 3
def test_publishes_failure_to_correct_url(goodserver):
publisher = HttpPublisher(name="Test",
success_url=goodserver.url + '/success',
failure_url=goodserver.url + '/failure',
publish_successes=True)
publisher.send(
Failure("failure", Validation("validation", priority=Priority.NORMAL),
"description"))
assert request_sent
assert requested_url == '/failure'
def _perform(validation, immutable_group_failures, results):
start = time.time()
try:
validation.perform(immutable_group_failures)
try:
runtime = validation.get_elapsed_time()
except NotImplementedError:
runtime = time.time() - start
if validation.timeout is not None and runtime > validation.timeout:
result = Failure(validation.name, validation,
"{} ran for {} (exceeded timeout of {})".format(
validation, runtime, validation.timeout),
time=runtime)
else:
result = Success(validation.name, validation,
time=runtime)
except Exception, e:
result = Failure(validation.name, validation, str(e),
time=time.time() - start)
def test_publish_stops_retrying(ratelimited):
global cutoff
cutoff = 4
global hits
hits = 0
pub = PagerDutyPublisher(ratelimited.url, "token")
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "message")
with pytest.raises(PublishFailure):
pub.send(failure)
def test_not_enough_attempts(failingserver):
global times_to_fail
times_to_fail = 3
publisher = HttpPublisher(name="Test", url=failingserver.url, attempts=3)
with pytest.raises(PublishFailure):
publisher.send(
Failure("failure",
Validation("validation", priority=Priority.NORMAL),
"description"))
def test_simple_email(httpserver, smtpserver):
email_pub = SimpleEmailPublisher({"real_name": "test", "address": "*****@*****.**"},
[{"real_name": "test", "address": "*****@*****.**"}],
host=smtpserver.addr[0], port=smtpserver.addr[1])
http_validator = \
HttpValidation.get(httpserver.url).expect_status_codes([200])
result = Failure("Check Status Route", http_validator,
description="Validation failure message")
email_pub.send(result)
assert len(smtpserver.outbox) == 1
payload = str(smtpserver.outbox[0].get_payload()[0])
assert payload.split("\n")[-1] == "Validation failure message"
def test_environment_name_is_present(httpserver):
environment = 'xanadu'
pub = PagerDutyPublisher(httpserver.url, "token", environment=environment)
message = pub._construct_message(
Failure(
"ternary computers not supported!",
Validation("bit frobnication validation",
priority=Priority.CRITICAL),
"unable to frobnicate bits!"))
assert message.startswith("Failure in %s:" % environment)
def test_timeout_too_short(slowserver):
global sleep_time
sleep_time = 2
publisher = HttpPublisher(name="Test",
url=slowserver.url + '/failure',
timeout_seconds=3)
publisher.send(
Failure("failure", Validation("validation", priority=Priority.NORMAL),
"description"))
assert request_sent
assert requested_url == '/failure'
def test_timeout_too_short(slowserver):
global sleep_time
sleep_time = 2
publisher = HttpPublisher(name="Test",
url=slowserver.url,
timeout_seconds=1)
with pytest.raises(PublishFailure):
publisher.send(
Failure("failure",
Validation("validation", priority=Priority.NORMAL),
"description"))
def test_construct_tree_failure():
pub = JUnitPublisher("should_not_be_created.xml")
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "unable to transmogrify", time=30)
tree = pub._construct_tree([failure]).getroot()
assert tree.get("failures") == str(1)
assert tree.get("tests") == str(1)
assert float(tree.get("time")) == 30
assert len(tree) == 1
for element in tree:
assert float(element.get("time")) == 30
for sub in element:
assert sub.text
def test_enough_attempts(failingserver):
global times_to_fail
times_to_fail = 2
publisher = HttpPublisher(name="Test",
url=failingserver.url + '/failure',
attempts=3)
publisher.send(
Failure("failure", Validation("validation", priority=Priority.NORMAL),
"description"))
assert request_sent
assert requested_url == '/failure'
def _compute_dry_run(validations, publishers):
"""Helper method for computing which validations are published where.
Provides programmatic access to the association between publishers
and validations. Return is of the form {publisher:[validation,...],...}.
"""
associations = {}
for publisher in publishers:
associations[publisher] = []
for validation in sorted(
validations, reverse=True,
key=lambda x: x.priority):
test_result = Failure(validation.name, validation, "failure")
if publisher.will_publish(test_result):
associations[publisher].append(validation)
return associations
def test_generate_id_reflexive():
pub = PagerDutyPublisher("url", "token")
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "unable to transmogrify")
another = Failure("foo", v, "unable")
assert pub._generate_id(failure) == pub._generate_id(another)
def test_failures_are_failures():
v = Validation("low", priority=Priority.LOW)
f = Failure("name", v, "desc")
assert f.is_failure()
def testSendFailure(no_post):
pager = new_publisher()
v = Validation("low", priority=Priority.CRITICAL)
failure = Failure("bar", v, "unable to transmogrify")
with pytest.raises(AttributeError):
pager.send(failure)
def test_failures_are_failures():
v = Validation("low", priority=Priority.LOW)
f = Failure("name", v, "desc")
assert f.is_failure()