def setUp(self):
self.couchdb_container = CouchDBContainer()
self.HOST = self.couchdb_container.couchdb_fqdn
self.DB = 'cookiejar-test'
_biscuit_tin.Timer = MagicMock()
super().setUp()
def setUp(self):
"""
Test set up:
* Build, if necessary, and start a CouchDB container and
connect as a BiscuitTin instance
* Start the HTTP API service on a free port, with the necessary
dependencies injected
* Create an HTTP client connection to the API service
"""
self.couchdb_container = CouchDBContainer()
# Configuration for Cookie Jar
self.HOST = self.couchdb_container.couchdb_fqdn
self.DB = 'elmo-test'
self.jar = BiscuitTin(self.HOST, self.DB, 1, timedelta(0))
# Configuration for HTTP service
self.API_PORT = get_open_port()
self.api = HTTP_API()
self.api.inject(APIDependency.CookieJar, self.jar)
self.api.inject(APIDependency.System, None)
self.api.listen(self.API_PORT)
self.http = HTTPConnection('localhost', self.API_PORT)
self.REQ_HEADER = {'Accept': 'application/json'}
# Block until service is up (or timeout)
start_time = finish_time = datetime.now()
service_up = False
while finish_time - start_time < timedelta(seconds=5):
response = None
try:
self.http.request('HEAD', '/')
response = self.http.getresponse()
except:
sleep(0.1)
finally:
self.http.close()
finish_time = datetime.now()
if isinstance(response, HTTPResponse):
service_up = True
break
if not service_up:
self.tearDown()
raise ConnectionError('Couldn\'t start API service in a reasonable amount of time')
class TestElmo(unittest.TestCase):
def setUp(self):
"""
Test set up:
* Build, if necessary, and start a CouchDB container and
connect as a BiscuitTin instance
* Start the HTTP API service on a free port, with the necessary
dependencies injected
* Create an HTTP client connection to the API service
"""
self.couchdb_container = CouchDBContainer()
# Configuration for Cookie Jar
self.HOST = self.couchdb_container.couchdb_fqdn
self.DB = 'elmo-test'
self.jar = BiscuitTin(self.HOST, self.DB, 1, timedelta(0))
# Configuration for HTTP service
self.API_PORT = get_open_port()
self.api = HTTP_API()
self.api.inject(APIDependency.CookieJar, self.jar)
self.api.inject(APIDependency.System, None)
self.api.listen(self.API_PORT)
self.http = HTTPConnection('localhost', self.API_PORT)
self.REQ_HEADER = {'Accept': 'application/json'}
# Block until service is up (or timeout)
start_time = finish_time = datetime.now()
service_up = False
while finish_time - start_time < timedelta(seconds=5):
response = None
try:
self.http.request('HEAD', '/')
response = self.http.getresponse()
except:
sleep(0.1)
finally:
self.http.close()
finish_time = datetime.now()
if isinstance(response, HTTPResponse):
service_up = True
break
if not service_up:
self.tearDown()
raise ConnectionError('Couldn\'t start API service in a reasonable amount of time')
def tearDown(self):
""" Tear down test set up """
self.http.close()
self.api.stop()
self.couchdb_container.tear_down()
def test_queue(self):
"""
HTTP API: GET /queue
"""
self.http.request('GET', '/queue', headers=self.REQ_HEADER)
r = self.http.getresponse()
self.assertEqual(r.status, 200)
self.assertEqual(r.headers.get_content_type(), 'application/json')
data = _decode_json_response(r)
self.assertIn('queue_length', data)
self.assertEqual(data['queue_length'], self.jar.queue_length()) # Should be 0
self.http.close()
# Add item to the queue
self.jar.mark_for_processing('/foo')
self.http.request('GET', '/queue', headers=self.REQ_HEADER)
data = _decode_json_response(self.http.getresponse())
self.assertEqual(data['queue_length'], self.jar.queue_length()) # Should be 1
def test_reprocess(self):
"""
HTTP API: POST /queue/reprocess
"""
# Add mocked update notifier to Cookie Jar
dirty_cookie_listener = MagicMock()
self.jar.add_listener(dirty_cookie_listener)
cookie_identifier = '/foo'
request = {'identifier': cookie_identifier}
self.http.request('POST', '/queue/reprocess', body=json.dumps(request), headers=self.REQ_HEADER)
r = self.http.getresponse()
self.assertEqual(r.status, 200)
self.assertEqual(r.headers.get_content_type(), 'application/json')
data = _decode_json_response(r)
self.assertEqual(data, request)
self.http.close()
# Check queue has been updated
self.assertEqual(self.jar.queue_length(), 1)
self.assertEqual(dirty_cookie_listener.call_count, 1)
@staticmethod
def _url_for_identifier(identifier:str):
""" URL for identifier """
if identifier[0] == "/":
return '/cookiejar?identifier={}'.format(identifier)
else:
return '/cookiejar/{}'.format(identifier)
def _fetch_test(self, identifier:str):
""" Generic fetch test """
source = 'foobar'
timestamp = datetime.now().replace(microsecond=0, tzinfo=timezone.utc)
metadata = Metadata({'foo': 123, 'bar': 'quux'})
enrichment = Enrichment(source, timestamp, metadata)
self.jar.enrich_cookie(identifier, enrichment)
self.http.request('GET', TestElmo._url_for_identifier(identifier), headers=self.REQ_HEADER)
r = self.http.getresponse()
self.assertEqual(r.status, 200)
self.assertEqual(r.headers.get_content_type(), 'application/json')
data = _decode_json_response(r)
fetched_identifier = data['identifier']
fetched_enrichment = json.loads(json.dumps(data['enrichments']), cls=EnrichmentJSONDecoder)[0]
self.assertEqual(fetched_identifier, identifier)
self.assertEqual(fetched_enrichment, enrichment)
def test_fetch_by_qs(self):
"""
HTTP API: GET /cookiejar?identifier=<identifier>
"""
self._fetch_test('/path/to/foo')
def test_fetch_by_route(self):
"""
HTTP API: GET /cookiejar/<identifier>
"""
self._fetch_test('foo_bar')
def _delete_test(self, identifier:str):
""" Generic delete test """
self.jar.mark_for_processing(identifier)
self.jar.mark_as_complete(identifier)
cookie = self.jar.fetch_cookie(identifier)
self.assertIsInstance(cookie, Cookie)
self.http.request('DELETE', TestElmo._url_for_identifier(identifier), headers=self.REQ_HEADER)
r = self.http.getresponse()
self.assertEqual(r.status, 200)
self.assertEqual(r.headers.get_content_type(), 'application/json')
data = _decode_json_response(r)
self.assertEqual(data, {'deleted':identifier})
deleted_cookie = self.jar.fetch_cookie(identifier)
self.assertIsNone(deleted_cookie)
def test_delete_by_qs(self):
"""
HTTP API: DELETE /cookiejar?identifier=<identifier>
"""
self._delete_test('/path/to/foo')
def test_delete_by_route(self):
"""
HTTP API: DELETE /cookiejar/<identifier>
"""
self._delete_test('foo_bar')
def test_thread_dump(self):
"""
HTTP API: GET /debug/threads
Note: This test only proves that the endpoint returns an OK
response and JSON data.
TODO At least validate the returned data's schema
"""
self.http.request('GET', '/debug/threads', headers=self.REQ_HEADER)
r = self.http.getresponse()
self.assertEqual(r.status, 200)
self.assertEqual(r.headers.get_content_type(), 'application/json')
class TestBiscuitTin(TestCookieJar):
"""
High-level integration and logic tests of the CookieJar-CouchDB
implementation (`BiscuitTin`). We assume that if the higher-level tests
pass and are suitably comprehensive, then the underlying levels of
abstraction are probably fine™
"""
def setUp(self):
self.couchdb_container = CouchDBContainer()
self.HOST = self.couchdb_container.couchdb_fqdn
self.DB = 'cookiejar-test'
_biscuit_tin.Timer = MagicMock()
super().setUp()
def tearDown(self):
self.couchdb_container.tear_down()
_biscuit_tin.Timer.reset_mock()
def _create_cookie_jar(self) -> BiscuitTin:
# TODO? We don't test the buffering (only the trivial case of a
# single document, zero-latency buffer)
return BiscuitTin(self.HOST, self.DB, 1, timedelta(0))
def _get_scheduled_fn(self) -> Callable[..., Any]:
return self.jar._broadcast
def _test_scheduling(self, expected_timeout:Real, expected_call:Callable[..., Any]):
_biscuit_tin.Timer.assert_called_with(expected_timeout, expected_call)
def _change_time(self, cookie_jar: CookieJar, change_time_to: int):
_biscuit_tin._now = MagicMock(return_value=change_time_to)
def test14_connection_failure(self):
"""
CookieJar Sequence: Enrich -> Reconnect -> Get Next
"""
self.jar.enrich_cookie(self.eg_identifiers[0], self.eg_enrichments[0])
new_jar = self._create_cookie_jar()
self.assertEqual(new_jar.queue_length(), 1)
to_process = new_jar.get_next_for_processing()
self.assertEqual(new_jar.queue_length(), 0)
self.assertIsInstance(to_process, Cookie)
self.assertEqual(to_process.identifier, self.eg_identifiers[0])
self.assertEqual(len(to_process.enrichments), 1)
self.assertEqual(to_process.enrichments[0], self.eg_enrichments[0])
def test15_connection_failure_while_processing(self):
"""
CookieJar Sequence: Enrich -> Get Next -> Reconnect -> Get Next
"""
self.jar.enrich_cookie(self.eg_identifiers[0], self.eg_enrichments[0])
before = self.jar.get_next_for_processing()
new_jar = self._create_cookie_jar()
self.assertEqual(new_jar.queue_length(), 1)
after = new_jar.get_next_for_processing()
self.assertEqual(before, after)
