def __init__(self, logger, download_database, storage_path, fernet,
vbms_client, queue, env_name):
self.logger = logger
self.download_database = download_database
self.storage_path = storage_path
self.fernet = fernet
self.vbms_client = vbms_client
self.queue = queue
self.env_name = env_name
self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
FilePath(__file__).parent().parent().child("templates").path, ),
autoescape=True)
self.document_types = DeferredValue()
def test_simple(self):
v = DeferredValue()
d = v.wait()
no_result(d)
v.completed(12)
assert success_result_of(d) == 12
assert success_result_of(v.wait()) == 12
def __init__(self, logger, download_database, storage_path, fernet,
vbms_client, queue, env_name):
self.logger = logger
self.download_database = download_database
self.storage_path = storage_path
self.fernet = fernet
self.vbms_client = vbms_client
self.queue = queue
self.env_name = env_name
self.jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(
FilePath(__file__).parent().parent().child("templates").path,
),
autoescape=True
)
self.document_types = DeferredValue()
class DownloadEFolder(object):
app = klein.Klein()
def __init__(self, logger, download_database, storage_path, fernet,
vbms_client, queue, env_name):
self.logger = logger
self.download_database = download_database
self.storage_path = storage_path
self.fernet = fernet
self.vbms_client = vbms_client
self.queue = queue
self.env_name = env_name
self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
FilePath(__file__).parent().parent().child("templates").path, ),
autoescape=True)
self.document_types = DeferredValue()
@classmethod
def from_config(cls, reactor, logger, queue, config_path):
with open(config_path) as f:
config = yaml.safe_load(f)
# TODO: bump this once alchimia properly handles pinning
thread_pool = ThreadPool(minthreads=1, maxthreads=1)
thread_pool.start()
reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)
return cls(
logger,
DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
FilePath(config["storage"]["filesystem"]),
fernet.MultiFernet(
[fernet.Fernet(key) for key in config["encryption_keys"]]),
VBMSClient(
reactor,
connect_vbms_path=config["connect_vbms"]["path"],
bundle_path=config["connect_vbms"]["bundle_path"],
endpoint_url=config["vbms"]["endpoint_url"],
keyfile=config["vbms"]["keyfile"],
samlfile=config["vbms"]["samlfile"],
key=config["vbms"].get("key"),
keypass=config["vbms"]["keypass"],
ca_cert=config["vbms"].get("ca_cert"),
client_cert=config["vbms"].get("client_cert"),
),
queue,
config["env"],
)
@classmethod
def create_demo(cls, reactor, logger):
return cls(logger=logger,
download_database=DemoMemoryDownloadDatabase(),
storage_path=None,
fernet=None,
vbms_client=None,
queue=None,
env_name="demo")
def render_template(self, template_name, data={}):
t = self.jinja_env.get_template(template_name)
return t.render(dict(data, env=self.env_name))
@inlineCallbacks
def start_download(self, file_number, request_id):
logger = self.logger.bind(file_number=file_number,
request_id=request_id)
logger.emit("list_documents.start")
try:
documents = yield self.vbms_client.list_documents(
logger, file_number)
except VBMSError as e:
logger.bind(
stdout=e.stdout,
stderr=e.stderr,
exit_code=e.exit_code,
).emit("list_documents.error")
yield self.download_database.mark_download_errored(
logger, request_id)
else:
logger.emit("list_documents.success")
documents = [
Document.from_json(request_id, doc) for doc in documents
]
yield self.download_database.create_documents(logger, documents)
for doc in documents:
self.queue.put(
functools.partial(self.start_file_download, logger, doc))
yield self.download_database.mark_download_manifest_downloaded(
logger, request_id)
@inlineCallbacks
def start_file_download(self, logger, document):
logger = logger.bind(document_id=document.document_id)
logger.emit("get_document.start")
try:
contents = yield self.vbms_client.fetch_document_contents(
logger, str(document.document_id))
except VBMSError as e:
logger.bind(
stdout=e.stdout,
stderr=e.stderr,
exit_code=e.exit_code,
).emit("get_document.error")
yield self.download_database.mark_document_errored(
logger, document)
else:
logger.emit("get_document.success")
target = self.storage_path.child(str(uuid.uuid4()))
target.setContent(self.fernet.encrypt(contents))
yield self.download_database.set_document_content_location(
logger, document, target.path)
@inlineCallbacks
def start_fetch_document_types(self):
document_types = yield self.vbms_client.get_document_types(self.logger)
self.document_types.completed(
{int(c["type_id"]): c["description"]
for c in document_types})
@inlineCallbacks
def queue_pending_work(self):
downloads, documents = yield self.download_database.get_pending_work(
self.logger)
for download in downloads:
self.queue.put(
functools.partial(
self.start_download,
download.file_number,
download.request_id,
))
for document in documents:
# TODO: self.logger doesn't include any info about the
# DownloadStatus. When triggered from the "usual" path it has
# ``file_number`` and ``request_id`` keys.
self.queue.put(
functools.partial(self.start_file_download, self.logger,
document))
@app.route("/")
@instrumented_route
def root(self, request):
request.redirect("/efolder-express/")
return succeed(None)
@app.route("/efolder-express/")
@instrumented_route
def index(self, request):
return self.render_template("index.html")
@app.route("/efolder-express/download/", methods=["POST"])
@instrumented_route
@inlineCallbacks
def download(self, request):
file_number = request.args["file_number"][0]
file_number = file_number.replace("-", "").replace(" ", "")
request_id = str(uuid.uuid4())
yield self.download_database.create_download(self.logger, request_id,
file_number)
self.queue.put(
functools.partial(self.start_download, file_number, request_id))
request.redirect("/efolder-express/download/{}/".format(request_id))
returnValue(None)
@app.route("/efolder-express/download/<request_id>/")
@instrumented_route
@inlineCallbacks
def download_status(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id)
returnValue(self.render_template("download.html",
{"status": download}))
@app.route("/efolder-express/download/<request_id>/json/")
@instrumented_route
@inlineCallbacks
def download_status_json(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id)
html = self.render_template("_download_status.html", {
"status": download,
})
request.setHeader("Content-Type", "application/json")
returnValue(
json.dumps({
"completed": download.completed or download.state == "ERRORED",
"html": html,
}))
@app.route("/efolder-express/download/<request_id>/zip/")
@instrumented_route
@inlineCallbacks
def download_zip(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id)
assert download.completed
self.logger.bind(
request_id=request_id,
file_number=download.file_number,
).emit("download")
document_types = yield self.document_types.wait()
path = download.build_zip(self.jinja_env, self.fernet, document_types)
request.setHeader(
"Content-Disposition",
"attachment; filename={}-eFolder.zip".format(download.file_number))
resource = File(path, defaultType="application/zip")
resource.isLeaf = True
request.notifyFinish().addBoth(lambda *args, **kwargs: os.remove(path))
returnValue(resource)
class DownloadEFolder(object):
app = klein.Klein()
def __init__(self, logger, download_database, storage_path, fernet,
vbms_client, queue, env_name):
self.logger = logger
self.download_database = download_database
self.storage_path = storage_path
self.fernet = fernet
self.vbms_client = vbms_client
self.queue = queue
self.env_name = env_name
self.jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(
FilePath(__file__).parent().parent().child("templates").path,
),
autoescape=True
)
self.document_types = DeferredValue()
@classmethod
def from_config(cls, reactor, logger, queue, config_path):
with open(config_path) as f:
config = yaml.safe_load(f)
# TODO: bump this once alchimia properly handles pinning
thread_pool = ThreadPool(minthreads=1, maxthreads=1)
thread_pool.start()
reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)
return cls(
logger,
DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
FilePath(config["storage"]["filesystem"]),
fernet.MultiFernet([
fernet.Fernet(key) for key in config["encryption_keys"]
]),
VBMSClient(
reactor,
connect_vbms_path=config["connect_vbms"]["path"],
bundle_path=config["connect_vbms"]["bundle_path"],
endpoint_url=config["vbms"]["endpoint_url"],
keyfile=config["vbms"]["keyfile"],
samlfile=config["vbms"]["samlfile"],
key=config["vbms"].get("key"),
keypass=config["vbms"]["keypass"],
ca_cert=config["vbms"].get("ca_cert"),
client_cert=config["vbms"].get("client_cert"),
),
queue,
config["env"],
)
@classmethod
def create_demo(cls, reactor, logger):
return cls(
logger=logger,
download_database=DemoMemoryDownloadDatabase(),
storage_path=None,
fernet=None,
vbms_client=None,
queue=None,
env_name="demo"
)
def render_template(self, template_name, data={}):
t = self.jinja_env.get_template(template_name)
return t.render(dict(data, env=self.env_name))
@inlineCallbacks
def start_download(self, file_number, request_id):
logger = self.logger.bind(
file_number=file_number, request_id=request_id
)
logger.emit("list_documents.start")
try:
documents = yield self.vbms_client.list_documents(
logger, file_number
)
except VBMSError as e:
logger.bind(
stdout=e.stdout,
stderr=e.stderr,
exit_code=e.exit_code,
).emit("list_documents.error")
yield self.download_database.mark_download_errored(
logger, request_id
)
else:
logger.emit("list_documents.success")
documents = [
Document.from_json(request_id, doc)
for doc in documents
]
yield self.download_database.create_documents(logger, documents)
for doc in documents:
self.queue.put(functools.partial(
self.start_file_download, logger, doc
))
yield self.download_database.mark_download_manifest_downloaded(
logger, request_id
)
@inlineCallbacks
def start_file_download(self, logger, document):
logger = logger.bind(document_id=document.document_id)
logger.emit("get_document.start")
try:
contents = yield self.vbms_client.fetch_document_contents(
logger, str(document.document_id)
)
except VBMSError as e:
logger.bind(
stdout=e.stdout,
stderr=e.stderr,
exit_code=e.exit_code,
).emit("get_document.error")
yield self.download_database.mark_document_errored(
logger, document
)
else:
logger.emit("get_document.success")
target = self.storage_path.child(str(uuid.uuid4()))
target.setContent(self.fernet.encrypt(contents))
yield self.download_database.set_document_content_location(
logger, document, target.path
)
@inlineCallbacks
def start_fetch_document_types(self):
document_types = yield self.vbms_client.get_document_types(self.logger)
self.document_types.completed({
int(c["type_id"]): c["description"]
for c in document_types
})
@inlineCallbacks
def queue_pending_work(self):
downloads, documents = yield self.download_database.get_pending_work(
self.logger
)
for download in downloads:
self.queue.put(functools.partial(
self.start_download, download.file_number, download.request_id,
))
for document in documents:
# TODO: self.logger doesn't include any info about the
# DownloadStatus. When triggered from the "usual" path it has
# ``file_number`` and ``request_id`` keys.
self.queue.put(functools.partial(
self.start_file_download, self.logger, document
))
@app.route("/")
@instrumented_route
def root(self, request):
request.redirect("/efolder-express/")
return succeed(None)
@app.route("/efolder-express/")
@instrumented_route
def index(self, request):
return self.render_template("index.html")
@app.route("/efolder-express/download/", methods=["POST"])
@instrumented_route
@inlineCallbacks
def download(self, request):
file_number = request.args["file_number"][0]
file_number = file_number.replace("-", "").replace(" ", "")
request_id = str(uuid.uuid4())
yield self.download_database.create_download(
self.logger, request_id, file_number
)
self.queue.put(functools.partial(
self.start_download, file_number, request_id
))
request.redirect("/efolder-express/download/{}/".format(request_id))
returnValue(None)
@app.route("/efolder-express/download/<request_id>/")
@instrumented_route
@inlineCallbacks
def download_status(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id
)
returnValue(self.render_template("download.html", {
"status": download
}))
@app.route("/efolder-express/download/<request_id>/json/")
@instrumented_route
@inlineCallbacks
def download_status_json(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id
)
html = self.render_template("_download_status.html", {
"status": download,
})
request.setHeader("Content-Type", "application/json")
returnValue(json.dumps({
"completed": download.completed or download.state == "ERRORED",
"html": html,
}))
@app.route("/efolder-express/download/<request_id>/zip/")
@instrumented_route
@inlineCallbacks
def download_zip(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id
)
assert download.completed
self.logger.bind(
request_id=request_id,
file_number=download.file_number,
).emit("download")
document_types = yield self.document_types.wait()
path = download.build_zip(self.jinja_env, self.fernet, document_types)
request.setHeader(
"Content-Disposition",
"attachment; filename={}-eFolder.zip".format(download.file_number)
)
resource = File(path, defaultType="application/zip")
resource.isLeaf = True
request.notifyFinish().addBoth(lambda *args, **kwargs: os.remove(path))
returnValue(resource)