def makeService(options): from twisted.internet import reactor assert not (options["config"] and options["demo"]) if options["demo"]: app = DownloadEFolder.create_demo(reactor, Logger(log)) else: queue = DeferredQueue() app = DownloadEFolder.from_config( reactor, Logger(log), queue, options["config"], ) if options.subCommand == "create-database": return CreateDatabaseService(reactor, app) if not options["demo"]: app.start_fetch_document_types() app.queue_pending_work() service = MultiService() endpoint = serverFromString(reactor, "tcp:8080:interface=127.0.0.1") StreamServerEndpointService( endpoint, Site(app.app.resource(), logPath="/dev/null"), ).setServiceParent(service) if not options["demo"]: for _ in xrange(8): DeferredQueueConsumerService( queue, lambda item: item()).setServiceParent(service) return service
def test_time(self): logger = Logger(FakeMemoryLog()) logger.time("test.event").stop() [msg] = logger._log.msgs assert set(msg) == {"event", "duration"} assert msg["event"] == "test.event" assert 0 < msg["duration"] < .1
def test_bind(self): logger = Logger(FakeMemoryLog()) logger.bind(key="value").emit("test.event1") logger.emit("test.event2") assert logger._log.msgs == [ {"key": "value", "event": "test.event1"}, {"event": "test.event2"}, ]
def test_get_pending_work_documents(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) d = db.mark_download_manifest_downloaded(logger, "test-request-id") success_result_of(d) doc = Document( id="test-document-id", download_id="test-request-id", document_id="{ABCD}", doc_type="00356", filename="file.pdf", received_at=datetime.datetime.utcnow(), source="CUI", content_location=None, errored=False, ) d = db.create_documents(logger, [doc]) success_result_of(d) d = db.get_pending_work(logger) downloads, documents = success_result_of(d) assert downloads == [] [document] = documents assert document.id == "test-document-id" d = db.set_document_content_location(logger, doc, "/path/to/content") success_result_of(d) d = db.get_pending_work(logger) assert success_result_of(d) == ([], [])
def test_set_document_content_location(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) doc = Document( id="test-document-id", download_id="test-request-id", document_id="{ABCD}", doc_type="00356", filename="file.pdf", received_at=datetime.datetime.utcnow(), source="CUI", content_location=None, errored=False, ) d = db.create_documents(logger, [doc]) success_result_of(d) d = db.set_document_content_location(logger, doc, "/path/to/content") success_result_of(d) download = success_result_of(db.get_download( logger, "test-request-id" )) assert download.completed assert download.percent_completed == 100 assert download.documents[0].content_location == "/path/to/content"
def test_create_download(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) assert self.scalar(db, db._downloads.select().count()) == 1 d = db.create_download(logger, "test-request-id-2", "987654321") success_result_of(d) assert self.scalar(db, db._downloads.select().count()) == 2
def test_mark_download_manifest_downloaded(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) d = db.mark_download_manifest_downloaded(logger, "test-request-id") success_result_of(d) download = success_result_of(db.get_download( logger, "test-request-id" )) assert download.state == "MANIFEST_DOWNLOADED"
def test_create_documents_empty(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) d = db.create_documents(logger, []) success_result_of(d) download = success_result_of(db.get_download( logger, "test-request-id" )) assert len(download.documents) == 0
def test_mark_download_errored(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) d = db.mark_download_errored(logger, "test-request-id") success_result_of(d) download = success_result_of(db.get_download( logger, "test-request-id" )) assert download.state == "ERRORED"
def test_get_download(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) download = success_result_of(db.get_download( logger, "test-request-id" )) assert download.request_id == "test-request-id" assert download.file_number == "123456789" assert download.state == "STARTED" assert download.documents == [] assert not download.completed assert download.percent_completed == 5
def test_get_pending_work_downloads(self, db): logger = Logger(FakeMemoryLog()) d = db.create_download(logger, "test-request-id", "123456789") success_result_of(d) d = db.get_pending_work(logger) downloads, documents = success_result_of(d) assert documents == [] [download] = downloads assert download.request_id == "test-request-id" d = db.mark_download_manifest_downloaded(logger, "test-request-id") success_result_of(d) d = db.get_pending_work(logger) assert success_result_of(d) == ([], [])
def test_bind(self): logger = Logger(FakeMemoryLog()) logger.bind(key="value").emit("test.event1") logger.emit("test.event2") assert logger._log.msgs == [ { "key": "value", "event": "test.event1" }, { "event": "test.event2" }, ]
def server(request, reactor): logger = Logger(FakeMemoryLog()) app = DownloadEFolder( logger=logger, download_database=DemoMemoryDownloadDatabase(), storage_path=None, fernet=None, vbms_client=None, queue=None, env_name="testing", ) endpoint = TCP4ServerEndpoint(reactor, 0) d = endpoint.listen(Site(app.app.resource(), logPath="/dev/null")) def addfinalizer(port): # Add a callback so that the server is shutdown at the end of the test. request.addfinalizer(port.stopListening) return port d.addCallback(addfinalizer) return pytest.blockon(d)
def test_emit(self): logger = Logger(FakeMemoryLog()) logger.emit("test.event") assert logger._log.msgs == [{"event": "test.event"}]
def test_get_download_non_existent(self, db): logger = Logger(FakeMemoryLog()) d = db.get_download(logger, "non-existent") with pytest.raises(DownloadNotFound): success_result_of(d)