def makeService(options):
from twisted.internet import reactor
assert not (options["config"] and options["demo"])
if options["demo"]:
app = DownloadEFolder.create_demo(reactor, Logger(log))
else:
queue = DeferredQueue()
app = DownloadEFolder.from_config(
reactor,
Logger(log),
queue,
options["config"],
)
if options.subCommand == "create-database":
return CreateDatabaseService(reactor, app)
if not options["demo"]:
app.start_fetch_document_types()
app.queue_pending_work()
service = MultiService()
endpoint = serverFromString(reactor, "tcp:8080:interface=127.0.0.1")
StreamServerEndpointService(
endpoint,
Site(app.app.resource(), logPath="/dev/null"),
).setServiceParent(service)
if not options["demo"]:
for _ in xrange(8):
DeferredQueueConsumerService(
queue, lambda item: item()).setServiceParent(service)
return service
def test_time(self):
logger = Logger(FakeMemoryLog())
logger.time("test.event").stop()
[msg] = logger._log.msgs
assert set(msg) == {"event", "duration"}
assert msg["event"] == "test.event"
assert 0 < msg["duration"] < .1
def test_time(self):
logger = Logger(FakeMemoryLog())
logger.time("test.event").stop()
[msg] = logger._log.msgs
assert set(msg) == {"event", "duration"}
assert msg["event"] == "test.event"
assert 0 < msg["duration"] < .1
def test_bind(self):
logger = Logger(FakeMemoryLog())
logger.bind(key="value").emit("test.event1")
logger.emit("test.event2")
assert logger._log.msgs == [
{"key": "value", "event": "test.event1"},
{"event": "test.event2"},
]
def test_get_pending_work_documents(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
d = db.mark_download_manifest_downloaded(logger, "test-request-id")
success_result_of(d)
doc = Document(
id="test-document-id",
download_id="test-request-id",
document_id="{ABCD}",
doc_type="00356",
filename="file.pdf",
received_at=datetime.datetime.utcnow(),
source="CUI",
content_location=None,
errored=False,
)
d = db.create_documents(logger, [doc])
success_result_of(d)
d = db.get_pending_work(logger)
downloads, documents = success_result_of(d)
assert downloads == []
[document] = documents
assert document.id == "test-document-id"
d = db.set_document_content_location(logger, doc, "/path/to/content")
success_result_of(d)
d = db.get_pending_work(logger)
assert success_result_of(d) == ([], [])
def test_set_document_content_location(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
doc = Document(
id="test-document-id",
download_id="test-request-id",
document_id="{ABCD}",
doc_type="00356",
filename="file.pdf",
received_at=datetime.datetime.utcnow(),
source="CUI",
content_location=None,
errored=False,
)
d = db.create_documents(logger, [doc])
success_result_of(d)
d = db.set_document_content_location(logger, doc, "/path/to/content")
success_result_of(d)
download = success_result_of(db.get_download(
logger, "test-request-id"
))
assert download.completed
assert download.percent_completed == 100
assert download.documents[0].content_location == "/path/to/content"
def test_create_download(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
assert self.scalar(db, db._downloads.select().count()) == 1
d = db.create_download(logger, "test-request-id-2", "987654321")
success_result_of(d)
assert self.scalar(db, db._downloads.select().count()) == 2
def test_mark_download_manifest_downloaded(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
d = db.mark_download_manifest_downloaded(logger, "test-request-id")
success_result_of(d)
download = success_result_of(db.get_download(
logger, "test-request-id"
))
assert download.state == "MANIFEST_DOWNLOADED"
def test_create_documents_empty(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
d = db.create_documents(logger, [])
success_result_of(d)
download = success_result_of(db.get_download(
logger, "test-request-id"
))
assert len(download.documents) == 0
def test_mark_download_errored(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
d = db.mark_download_errored(logger, "test-request-id")
success_result_of(d)
download = success_result_of(db.get_download(
logger, "test-request-id"
))
assert download.state == "ERRORED"
def test_get_download(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
download = success_result_of(db.get_download(
logger, "test-request-id"
))
assert download.request_id == "test-request-id"
assert download.file_number == "123456789"
assert download.state == "STARTED"
assert download.documents == []
assert not download.completed
assert download.percent_completed == 5
def test_get_pending_work_downloads(self, db):
logger = Logger(FakeMemoryLog())
d = db.create_download(logger, "test-request-id", "123456789")
success_result_of(d)
d = db.get_pending_work(logger)
downloads, documents = success_result_of(d)
assert documents == []
[download] = downloads
assert download.request_id == "test-request-id"
d = db.mark_download_manifest_downloaded(logger, "test-request-id")
success_result_of(d)
d = db.get_pending_work(logger)
assert success_result_of(d) == ([], [])
def test_bind(self):
logger = Logger(FakeMemoryLog())
logger.bind(key="value").emit("test.event1")
logger.emit("test.event2")
assert logger._log.msgs == [
{
"key": "value",
"event": "test.event1"
},
{
"event": "test.event2"
},
]
def server(request, reactor):
logger = Logger(FakeMemoryLog())
app = DownloadEFolder(
logger=logger,
download_database=DemoMemoryDownloadDatabase(),
storage_path=None,
fernet=None,
vbms_client=None,
queue=None,
env_name="testing",
)
endpoint = TCP4ServerEndpoint(reactor, 0)
d = endpoint.listen(Site(app.app.resource(), logPath="/dev/null"))
def addfinalizer(port):
# Add a callback so that the server is shutdown at the end of the test.
request.addfinalizer(port.stopListening)
return port
d.addCallback(addfinalizer)
return pytest.blockon(d)
def test_emit(self):
logger = Logger(FakeMemoryLog())
logger.emit("test.event")
assert logger._log.msgs == [{"event": "test.event"}]
def test_emit(self):
logger = Logger(FakeMemoryLog())
logger.emit("test.event")
assert logger._log.msgs == [{"event": "test.event"}]
def test_get_download_non_existent(self, db):
logger = Logger(FakeMemoryLog())
d = db.get_download(logger, "non-existent")
with pytest.raises(DownloadNotFound):
success_result_of(d)
