def test_get_events_no_callback(self):
"""
When a request is made to Marathon's event stream, a callback should
not receive event data if there is no callback for the event type.
"""
data = []
d = self.cleanup_d(self.client.get_events({'test': data.append}))
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url=self.uri('/v2/events'),
query={'event_type': ['test']}))
self.assertThat(request.requestHeaders,
HasHeader('accept', ['text/event-stream']))
request.setResponseCode(200)
request.setHeader('Content-Type', 'text/event-stream')
json_data = {'hello': 'world'}
request.write(b'event: not_test\n')
request.write(
'data: {}\n'.format(json.dumps(json_data)).encode('utf-8'))
request.write(b'\n')
yield wait0()
self.assertThat(data, Equals([]))
request.finish()
yield d
# Expect request.finish() to result in a logged failure
flush_logged_errors(ResponseDone)
def test_timer_errors(self):
"""
If the timed check fails (for example, because registration fails), the
error should be caught and logged.
"""
with AcmeFixture(client=FailingClient()) as fixture:
# Registration is triggered with service starts.
fixture.service.startService()
latest_logs = flush_logged_errors()
self.assertThat(latest_logs, HasLength(1))
self.assertThat(
str(latest_logs[0]), Contains('Failing at "register".'))
# Forcing a check will trigger again the registration.
self.assertThat(
fixture.service._check_certs(),
succeeded(Always()))
latest_logs = flush_logged_errors()
self.assertThat(latest_logs, HasLength(1))
self.assertThat(
str(latest_logs[0]), Contains('Failing at "register".'))
# Manually stop the service to not stop it from the fixture
# and trigger another failure.
self.assertThat(
fixture.service.stopService(),
failed(AfterPreprocessing(
lambda f: f.value.args[0], Equals('Failing at "stop".'))))
latest_logs = flush_logged_errors()
def test_get_events(self):
"""
When a request is made to Marathon's event stream, a callback should
receive JSON-decoded data before the connection is closed.
"""
data = []
d = self.cleanup_d(self.client.get_events({'test': data.append}))
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url=self.uri('/v2/events')))
self.assertThat(request.requestHeaders,
HasHeader('accept', ['text/event-stream']))
request.setResponseCode(200)
request.setHeader('Content-Type', 'text/event-stream')
json_data = {'hello': 'world'}
request.write(b'event: test\n')
request.write(b'data: %s\n' % (json.dumps(json_data).encode('utf-8'),))
request.write(b'\n')
yield wait0()
self.assertThat(data, Equals([json_data]))
request.finish()
yield d
# Expect request.finish() to result in a logged failure
flush_logged_errors(ResponseDone)
def test_storage_dir_provided(self):
"""
When the program is run with an argument, it should start up and run.
The program is expected to fail because it is unable to connect to
Marathon.
This test takes a while because we have to let txacme go through it's
initial sync (registration + issuing of 0 certificates) before things
can be halted.
"""
temp_dir = self.useFixture(TempDir())
yield main(
reactor,
raw_args=[
temp_dir.path,
'--acme',
LETSENCRYPT_STAGING_DIRECTORY.asText(),
'--marathon',
'http://localhost:28080' # An address we can't reach
])
# Expect a 'certs' directory to be created
self.assertThat(os.path.isdir(temp_dir.join('certs')), Equals(True))
# Expect a default certificate to be created
self.assertThat(os.path.isfile(temp_dir.join('default.pem')),
Equals(True))
# Expect to be unable to connect to Marathon
flush_logged_errors(ConnectionRefusedError)
def test_storage_dir_provided(self):
"""
When the program is run with an argument, it should start up and run.
The program is expected to fail because it is unable to connect to
Marathon.
This test takes a while because we have to let txacme go through it's
initial sync (registration + issuing of 0 certificates) before things
can be halted.
"""
temp_dir = self.useFixture(TempDir())
yield main(reactor, raw_args=[
temp_dir.path,
'--acme', LETSENCRYPT_STAGING_DIRECTORY.asText(),
'--marathon', 'http://localhost:28080' # An address we can't reach
])
# Expect a 'certs' directory to be created
self.assertThat(os.path.isdir(temp_dir.join('certs')), Equals(True))
# Expect an 'unmanaged-certs' directory to be created
self.assertThat(
os.path.isdir(temp_dir.join('unmanaged-certs')), Equals(True))
# Expect a default certificate to be created
self.assertThat(os.path.isfile(temp_dir.join('default.pem')),
Equals(True))
# Expect to be unable to connect to Marathon
flush_logged_errors(ConnectionRefusedError)
def test_failure_during_request(self):
"""
When a failure occurs during a request, the exception is propagated
to the request's deferred.
"""
client = self.get_client(failing_client)
d = client.request('GET', path='/hello')
self.assertThat(d, failed(MatchesStructure(
value=IsInstance(RuntimeError))))
flush_logged_errors(RuntimeError)
def test_failure_during_request(self):
"""
When a failure occurs during a request, the exception is propagated
to the request's deferred.
"""
client = self.get_client(failing_client)
d = client.request('GET', path='/hello')
self.assertThat(d, failed(MatchesStructure(
value=IsInstance(RuntimeError))))
flush_logged_errors(RuntimeError)
def test_errors(self):
"""
If a cert renewal fails within the panic interval, the panic callback
is invoked; otherwise the error is logged normally.
"""
now = datetime(2000, 1, 1, 0, 0, 0)
certs = {
u'a' * 100:
_generate_cert(u'example.com',
not_valid_before=now - timedelta(seconds=1),
not_valid_after=now + timedelta(days=31)),
}
panics = []
with AcmeFixture(now=now,
certs=certs,
panic=lambda *a: panics.append(a)) as fixture:
fixture.service.startService()
self.assertThat(fixture.service.when_certs_valid(),
succeeded(Is(None)))
self.assertThat(fixture.responder.challenges, HasLength(0))
fixture.clock.advance(36 * 60 * 60)
self.assertThat(flush_logged_errors(), HasLength(1))
self.assertThat(panics, Equals([]))
self.assertThat(fixture.responder.challenges, HasLength(0))
fixture.clock.advance(15 * 24 * 60 * 60)
self.assertThat(
panics,
MatchesListwise([
MatchesListwise([IsInstance(Failure),
Equals(u'a' * 100)]),
]))
self.assertThat(fixture.responder.challenges, HasLength(0))
def test_errors(self):
"""
If a cert renewal fails within the panic interval, the panic callback
is invoked; otherwise the error is logged normally.
"""
now = datetime(2000, 1, 1, 0, 0, 0)
certs = {
u'a' * 100: _generate_cert(
u'example.com',
not_valid_before=now - timedelta(seconds=1),
not_valid_after=now + timedelta(days=31)),
}
panics = []
with AcmeFixture(now=now, certs=certs,
panic=lambda *a: panics.append(a)) as fixture:
fixture.service.startService()
self.assertThat(
fixture.service.when_certs_valid(),
succeeded(Is(None)))
self.assertThat(fixture.responder.challenges, HasLength(0))
fixture.clock.advance(36 * 60 * 60)
self.assertThat(flush_logged_errors(), HasLength(1))
self.assertThat(panics, Equals([]))
self.assertThat(fixture.responder.challenges, HasLength(0))
fixture.clock.advance(15 * 24 * 60 * 60)
self.assertThat(
panics,
MatchesListwise([
MatchesListwise([IsInstance(Failure), Equals(u'a' * 100)]),
]))
self.assertThat(fixture.responder.challenges, HasLength(0))
def test_default_panic(self):
"""
The default panic callback logs a message via ``twisted.logger``.
"""
try:
1 / 0
except:
f = Failure()
_default_panic(f, u'server_name')
self.assertThat(flush_logged_errors(), Equals([f]))
def test_timer_errors(self):
"""
If the timed check fails (for example, because registration fails), the
error should be caught and logged.
"""
with AcmeFixture(client=FailingClient()) as fixture:
fixture.service.startService()
self.assertThat(fixture.service._check_certs(),
succeeded(Always()))
self.assertThat(flush_logged_errors(), HasLength(2))
def test_default_panic(self):
"""
The default panic callback logs a message via ``twisted.logger``.
"""
try:
1 / 0
except BaseException:
f = Failure()
_default_panic(f, u'server_name')
self.assertThat(flush_logged_errors(), Equals([f]))
def test_request_partial_failure(self):
"""
When a request is made and an error status code is returned from some
(but not all) of the matathon-lb instances, then the request returns
the list of responses with a None value for the unhappy request.
"""
d = self.cleanup_d(self.client.request('GET', path='/my-path'))
lb1_request = yield self.requests.get()
self.assertThat(lb1_request, HasRequestProperties(
method='GET', url='http://lb1:9090/my-path'))
lb2_request = yield self.requests.get()
self.assertThat(lb2_request, HasRequestProperties(
method='GET', url='http://lb2:9090/my-path'))
# Fail the first one
lb1_request.setResponseCode(500)
lb1_request.setHeader('content-type', 'text/plain')
lb1_request.write(b'Internal Server Error')
lb1_request.finish()
# ...but succeed the second
lb2_request.setResponseCode(200)
lb2_request.setHeader('content-type', 'text/plain')
lb2_request.write(b'Yes, I work')
lb2_request.finish()
responses = yield d
self.assertThat(responses, HasLength(2))
lb1_response, lb2_response = responses
self.assertThat(lb1_response, Is(None))
self.assertThat(lb2_response, MatchesStructure(
code=Equals(200),
headers=HasHeader('content-type', ['text/plain'])
))
lb2_response_content = yield lb2_response.content()
self.assertThat(lb2_response_content, Equals(b'Yes, I work'))
flush_logged_errors(HTTPError)
def test_request_partial_failure(self):
"""
When a request is made and an error status code is returned from some
(but not all) of the matathon-lb instances, then the request returns
the list of responses with a None value for the unhappy request.
"""
d = self.cleanup_d(self.client.request('GET', path='/my-path'))
lb1_request = yield self.requests.get()
self.assertThat(lb1_request, HasRequestProperties(
method='GET', url='http://lb1:9090/my-path'))
lb2_request = yield self.requests.get()
self.assertThat(lb2_request, HasRequestProperties(
method='GET', url='http://lb2:9090/my-path'))
# Fail the first one
lb1_request.setResponseCode(500)
lb1_request.setHeader('content-type', 'text/plain')
lb1_request.write(b'Internal Server Error')
lb1_request.finish()
# ...but succeed the second
lb2_request.setResponseCode(200)
lb2_request.setHeader('content-type', 'text/plain')
lb2_request.write(b'Yes, I work')
lb2_request.finish()
responses = yield d
self.assertThat(responses, HasLength(2))
lb1_response, lb2_response = responses
self.assertThat(lb1_response, Is(None))
self.assertThat(lb2_response, MatchesStructure(
code=Equals(200),
headers=HasHeader('content-type', ['text/plain'])
))
lb2_response_content = yield lb2_response.content()
self.assertThat(lb2_response_content, Equals(b'Yes, I work'))
flush_logged_errors(HTTPError)
def test_request_fallback_all_failed(self):
"""
When we make a request and there are multiple Marathon endpoints
specified, and all the endpoints fail, the last failure should be
returned.
"""
agent = PerLocationAgent()
agent.add_agent(b'localhost:8080', FailingAgent(RuntimeError('8080')))
agent.add_agent(b'localhost:9090', FailingAgent(RuntimeError('9090')))
client = MarathonClient(
['http://localhost:8080', 'http://localhost:9090'],
client=treq_HTTPClient(agent))
d = self.cleanup_d(client.request('GET', path='/my-path'))
yield wait0()
self.assertThat(d, failed(WithErrorTypeAndMessage(
RuntimeError, '9090')))
flush_logged_errors(RuntimeError)
def test_request_fallback_all_failed(self):
"""
When we make a request and there are multiple Marathon endpoints
specified, and all the endpoints fail, the last failure should be
returned.
"""
agent = PerLocationAgent()
agent.add_agent(b'localhost:8080', FailingAgent(RuntimeError('8080')))
agent.add_agent(b'localhost:9090', FailingAgent(RuntimeError('9090')))
client = MarathonClient(
['http://localhost:8080', 'http://localhost:9090'],
client=treq_HTTPClient(agent))
d = self.cleanup_d(client.request('GET', path='/my-path'))
yield wait0()
self.assertThat(d, failed(WithErrorTypeAndMessage(
RuntimeError, '9090')))
flush_logged_errors(RuntimeError)
def test_timer_errors(self):
"""
If the timed check fails (for example, because registration fails), the
error should be caught and logged.
"""
with AcmeFixture(client=FailingClient()) as fixture:
fixture.service.startService()
self.assertThat(
fixture.service._check_certs(),
succeeded(Always()))
self.assertThat(flush_logged_errors(), HasLength(2))
def test_get_events_multiple_callbacks(self):
"""
When a request is made to Marathon's event stream, and there are
events for multiple callbacks, those callbacks should receive
JSON-decoded data for each event.
"""
data1 = []
data2 = []
d = self.cleanup_d(self.client.get_events({
'test1': data1.append,
'test2': data2.append
}))
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url=self.uri('/v2/events')))
self.assertThat(request.requestHeaders,
HasHeader('accept', ['text/event-stream']))
request.setResponseCode(200)
request.setHeader('Content-Type', 'text/event-stream')
json_data1 = {'hello': 'world'}
request.write(b'event: test1\n')
request.write(b'data: %s\n' % (json.dumps(json_data1).encode('utf-8')))
request.write(b'\n')
json_data2 = {'hello': 'computer'}
request.write(b'event: test2\n')
request.write(b'data: %s\n' % (json.dumps(json_data2).encode('utf-8')))
request.write(b'\n')
yield wait0()
self.assertThat(data1, Equals([json_data1]))
self.assertThat(data2, Equals([json_data2]))
request.finish()
yield d
# Expect request.finish() to result in a logged failure
flush_logged_errors(ResponseDone)
def test_cannot_listen(self):
"""
When the program is run with an argument and a listen address specified
with a port that we can't listen on (e.g. port 1), a CannotListenError
is expected to be logged and the program should stop.
"""
temp_dir = self.useFixture(TempDir())
yield main(reactor, raw_args=[
temp_dir.path,
'--listen', ':1', # A port we can't listen on
])
# Expect a 'certs' directory to be created
self.assertThat(os.path.isdir(temp_dir.join('certs')), Equals(True))
# Expect a default certificate to be created
self.assertThat(os.path.isfile(temp_dir.join('default.pem')),
Equals(True))
# Expect to be unable to listen
flush_logged_errors(CannotListenError)
def test_unexpectedErrorAndNotificationFailure(self):
# If unexpectedError is called while a notification is pending and the
# notification subsequently fails, the first failure "wins" and is
# passed on to the termination deferred.
deferred = defer.Deferred()
self.protocol.runNotification(lambda: deferred)
self.protocol.unexpectedError(makeFailure(TypeError))
runtime_error_failure = makeFailure(RuntimeError)
deferred.errback(runtime_error_failure)
self.assertEqual(
flush_logged_errors(RuntimeError), [runtime_error_failure])
return assert_fails_with(
self.termination_deferred, TypeError)
def test_get_events_multiple_events(self):
"""
When a request is made to Marathon's event stream, and there are
multiple events for a single callback, that callback should receive
JSON-decoded data for each event.
"""
data = []
d = self.cleanup_d(self.client.get_events({'test': data.append}))
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url=self.uri('/v2/events'),
query={'event_type': ['test']}))
self.assertThat(request.requestHeaders,
HasHeader('accept', ['text/event-stream']))
request.setResponseCode(200)
request.setHeader('Content-Type', 'text/event-stream')
json_data1 = {'hello': 'world'}
request.write(b'event: test\n')
request.write(
'data: {}\n'.format(json.dumps(json_data1)).encode('utf-8'))
request.write(b'\n')
json_data2 = {'hi': 'planet'}
request.write(
'data: {}\n'.format(json.dumps(json_data2)).encode('utf-8'))
request.write(b'event: test\n')
request.write(b'\n')
yield wait0()
self.assertThat(data, Equals([json_data1, json_data2]))
request.finish()
yield d
# Expect request.finish() to result in a logged failure
flush_logged_errors(ResponseDone)
def test_get_events_multiple_callbacks(self):
"""
When a request is made to Marathon's event stream, and there are
events for multiple callbacks, those callbacks should receive
JSON-decoded data for each event.
"""
data1 = []
data2 = []
d = self.cleanup_d(self.client.get_events({
'test1': data1.append,
'test2': data2.append
}))
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url=self.uri('/v2/events'),
query={'event_type': ['test1', 'test2']}))
self.assertThat(request.requestHeaders,
HasHeader('accept', ['text/event-stream']))
request.setResponseCode(200)
request.setHeader('Content-Type', 'text/event-stream')
json_data1 = {'hello': 'world'}
write_json_event(request, 'test1', json_data1)
json_data2 = {'hello': 'computer'}
write_json_event(request, 'test2', json_data2)
yield wait0()
self.assertThat(data1, Equals([json_data1]))
self.assertThat(data2, Equals([json_data2]))
request.finish()
yield d
# Expect request.finish() to result in a logged failure
flush_logged_errors(ResponseDone)
def test_uncleanExitAndPendingNotificationFails(self):
# If the process exits with a non-zero exit code while a
# notification is pending and the notification subsequently
# fails, the ProcessTerminated is still passed on to the
# termination deferred.
deferred = defer.Deferred()
self.protocol.runNotification(lambda: deferred)
self.simulateProcessExit(clean=False)
runtime_error_failure = makeFailure(RuntimeError)
deferred.errback(runtime_error_failure)
self.assertEqual(
flush_logged_errors(RuntimeError), [runtime_error_failure])
return assert_fails_with(
self.termination_deferred, error.ProcessTerminated)
def test_request_failure(self):
"""
When the requests to all the marathon-lb instances have a bad status
code then an error should be raised.
"""
d = self.cleanup_d(self.client.request('GET', path='/my-path'))
for lb in ['lb1', 'lb2']:
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url='http://%s:9090/my-path' % (lb,)))
request.setResponseCode(500)
request.setHeader('content-type', 'text/plain')
request.write(b'Internal Server Error')
request.finish()
yield wait0()
self.assertThat(d, failed(WithErrorTypeAndMessage(
RuntimeError,
'Failed to make a request to all marathon-lb instances'
)))
flush_logged_errors(HTTPError)
def test_cannot_listen(self):
"""
When the program is run with an argument and a listen address specified
with a port that we can't listen on (e.g. port 1), a CannotListenError
is expected to be logged and the program should stop.
"""
temp_dir = self.useFixture(TempDir())
yield main(
reactor,
raw_args=[
temp_dir.path,
'--listen',
':1', # A port we can't listen on
])
# Expect a 'certs' directory to be created
self.assertThat(os.path.isdir(temp_dir.join('certs')), Equals(True))
# Expect a default certificate to be created
self.assertThat(os.path.isfile(temp_dir.join('default.pem')),
Equals(True))
# Expect to be unable to listen
flush_logged_errors(CannotListenError)
def test_request_fallback(self):
"""
When we make a request and there are multiple Marathon endpoints
specified, and an endpoint fails, the next endpoint is used.
"""
agent = PerLocationAgent()
agent.add_agent(b'localhost:8080', FailingAgent())
agent.add_agent(b'localhost:9090', self.fake_server.get_agent())
client = MarathonClient(
['http://localhost:8080', 'http://localhost:9090'],
client=treq_HTTPClient(agent))
d = self.cleanup_d(client.request('GET', path='/my-path'))
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url='http://localhost:9090/my-path'))
request.setResponseCode(200)
request.finish()
yield d
flush_logged_errors(RuntimeError)
def test_request_fallback(self):
"""
When we make a request and there are multiple Marathon endpoints
specified, and an endpoint fails, the next endpoint is used.
"""
agent = PerLocationAgent()
agent.add_agent(b'localhost:8080', FailingAgent())
agent.add_agent(b'localhost:9090', self.fake_server.get_agent())
client = MarathonClient(
['http://localhost:8080', 'http://localhost:9090'],
client=treq_HTTPClient(agent))
d = self.cleanup_d(client.request('GET', path='/my-path'))
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url='http://localhost:9090/my-path'))
request.setResponseCode(200)
request.finish()
yield d
flush_logged_errors(RuntimeError)
def test_request_failure(self):
"""
When the requests to all the marathon-lb instances have a bad status
code then an error should be raised.
"""
d = self.cleanup_d(self.client.request('GET', path='/my-path'))
for lb in ['lb1', 'lb2']:
request = yield self.requests.get()
self.assertThat(request, HasRequestProperties(
method='GET', url='http://%s:9090/my-path' % (lb,)))
request.setResponseCode(500)
request.setHeader('content-type', 'text/plain')
request.write(b'Internal Server Error')
request.finish()
yield wait0()
self.assertThat(d, failed(WithErrorTypeAndMessage(
RuntimeError,
'Failed to make a request to all marathon-lb instances'
)))
flush_logged_errors(HTTPError)
def test_errorBeforeStatusReportAndFailingMirrorFailed(self):
# If the subprocess exits before reporting success or failure, *and*
# the attempt to record failure fails, there's not much we can do but
# we should still not hang. In keeping with the general policy, we
# fire the termination deferred with the first thing to go wrong --
# the process termination in this case -- and log.err() the failed
# attempt to call mirrorFailed().
runtime_error_failure = makeFailure(RuntimeError)
class FailingMirrorFailedStubPullerListener(self.StubPullerListener):
def mirrorFailed(self, message, oops):
return runtime_error_failure
self.protocol.listener = FailingMirrorFailedStubPullerListener()
self.listener = self.protocol.listener
self.protocol.errReceived('traceback')
self.simulateProcessExit(clean=False)
self.assertEqual(
flush_logged_errors(RuntimeError), [runtime_error_failure])
return assert_fails_with(
self.termination_deferred, error.ProcessTerminated)
def check_finishJob_called(result):
self.assertEqual([('finishJobID', job_id, 'SUCCESS', '')],
worker_monitor.codeimport_endpoint.calls)
errors = flush_logged_errors(Fail)
self.assertEqual(1, len(errors))
