def start_download(self, file_number, request_id): logger = self.logger.bind( file_number=file_number, request_id=request_id ) logger.emit("list_documents.start") try: documents = yield self.vbms_client.list_documents( logger, file_number ) except VBMSError as e: logger.bind( stdout=e.stdout, stderr=e.stderr, exit_code=e.exit_code, ).emit("list_documents.error") yield self.download_database.mark_download_errored( logger, request_id ) else: logger.emit("list_documents.success") documents = [ Document.from_json(request_id, doc) for doc in documents ] yield self.download_database.create_documents(logger, documents) for doc in documents: self.queue.put(functools.partial( self.start_file_download, logger, doc )) yield self.download_database.mark_download_manifest_downloaded( logger, request_id )
def start_download(self, file_number, request_id): logger = self.logger.bind(file_number=file_number, request_id=request_id) logger.emit("list_documents.start") try: documents = yield self.vbms_client.list_documents( logger, file_number) except VBMSError as e: logger.bind( stdout=e.stdout, stderr=e.stderr, exit_code=e.exit_code, ).emit("list_documents.error") yield self.download_database.mark_download_errored( logger, request_id) else: logger.emit("list_documents.success") documents = [ Document.from_json(request_id, doc) for doc in documents ] yield self.download_database.create_documents(logger, documents) for doc in documents: self.queue.put( functools.partial(self.start_file_download, logger, doc)) yield self.download_database.mark_download_manifest_downloaded( logger, request_id)
def test_from_json(self): doc = Document.from_json("test-document-id", { "document_id": "123456789", "doc_type": "123", "filename": "abc.pdf", "received_at": None, "source": "The moon", }) assert doc.received_at is None