예제 #1
0
    def __init__(self, logger, download_database, storage_path, fernet,
                 vbms_client, queue, env_name):
        self.logger = logger
        self.download_database = download_database
        self.storage_path = storage_path
        self.fernet = fernet
        self.vbms_client = vbms_client
        self.queue = queue
        self.env_name = env_name

        self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
            FilePath(__file__).parent().parent().child("templates").path, ),
                                            autoescape=True)
        self.document_types = DeferredValue()
예제 #2
0
    def test_simple(self):
        v = DeferredValue()

        d = v.wait()
        no_result(d)

        v.completed(12)
        assert success_result_of(d) == 12

        assert success_result_of(v.wait()) == 12
    def __init__(self, logger, download_database, storage_path, fernet,
                 vbms_client, queue, env_name):
        self.logger = logger
        self.download_database = download_database
        self.storage_path = storage_path
        self.fernet = fernet
        self.vbms_client = vbms_client
        self.queue = queue
        self.env_name = env_name

        self.jinja_env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(
                FilePath(__file__).parent().parent().child("templates").path,
            ),
            autoescape=True
        )
        self.document_types = DeferredValue()
예제 #4
0
class DownloadEFolder(object):
    app = klein.Klein()

    def __init__(self, logger, download_database, storage_path, fernet,
                 vbms_client, queue, env_name):
        self.logger = logger
        self.download_database = download_database
        self.storage_path = storage_path
        self.fernet = fernet
        self.vbms_client = vbms_client
        self.queue = queue
        self.env_name = env_name

        self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
            FilePath(__file__).parent().parent().child("templates").path, ),
                                            autoescape=True)
        self.document_types = DeferredValue()

    @classmethod
    def from_config(cls, reactor, logger, queue, config_path):
        with open(config_path) as f:
            config = yaml.safe_load(f)

        # TODO: bump this once alchimia properly handles pinning
        thread_pool = ThreadPool(minthreads=1, maxthreads=1)
        thread_pool.start()
        reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)

        return cls(
            logger,
            DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
            FilePath(config["storage"]["filesystem"]),
            fernet.MultiFernet(
                [fernet.Fernet(key) for key in config["encryption_keys"]]),
            VBMSClient(
                reactor,
                connect_vbms_path=config["connect_vbms"]["path"],
                bundle_path=config["connect_vbms"]["bundle_path"],
                endpoint_url=config["vbms"]["endpoint_url"],
                keyfile=config["vbms"]["keyfile"],
                samlfile=config["vbms"]["samlfile"],
                key=config["vbms"].get("key"),
                keypass=config["vbms"]["keypass"],
                ca_cert=config["vbms"].get("ca_cert"),
                client_cert=config["vbms"].get("client_cert"),
            ),
            queue,
            config["env"],
        )

    @classmethod
    def create_demo(cls, reactor, logger):
        return cls(logger=logger,
                   download_database=DemoMemoryDownloadDatabase(),
                   storage_path=None,
                   fernet=None,
                   vbms_client=None,
                   queue=None,
                   env_name="demo")

    def render_template(self, template_name, data={}):
        t = self.jinja_env.get_template(template_name)
        return t.render(dict(data, env=self.env_name))

    @inlineCallbacks
    def start_download(self, file_number, request_id):
        logger = self.logger.bind(file_number=file_number,
                                  request_id=request_id)

        logger.emit("list_documents.start")
        try:
            documents = yield self.vbms_client.list_documents(
                logger, file_number)
        except VBMSError as e:
            logger.bind(
                stdout=e.stdout,
                stderr=e.stderr,
                exit_code=e.exit_code,
            ).emit("list_documents.error")
            yield self.download_database.mark_download_errored(
                logger, request_id)
        else:
            logger.emit("list_documents.success")

            documents = [
                Document.from_json(request_id, doc) for doc in documents
            ]
            yield self.download_database.create_documents(logger, documents)
            for doc in documents:
                self.queue.put(
                    functools.partial(self.start_file_download, logger, doc))
            yield self.download_database.mark_download_manifest_downloaded(
                logger, request_id)

    @inlineCallbacks
    def start_file_download(self, logger, document):
        logger = logger.bind(document_id=document.document_id)
        logger.emit("get_document.start")

        try:
            contents = yield self.vbms_client.fetch_document_contents(
                logger, str(document.document_id))
        except VBMSError as e:
            logger.bind(
                stdout=e.stdout,
                stderr=e.stderr,
                exit_code=e.exit_code,
            ).emit("get_document.error")
            yield self.download_database.mark_document_errored(
                logger, document)
        else:
            logger.emit("get_document.success")
            target = self.storage_path.child(str(uuid.uuid4()))
            target.setContent(self.fernet.encrypt(contents))
            yield self.download_database.set_document_content_location(
                logger, document, target.path)

    @inlineCallbacks
    def start_fetch_document_types(self):
        document_types = yield self.vbms_client.get_document_types(self.logger)
        self.document_types.completed(
            {int(c["type_id"]): c["description"]
             for c in document_types})

    @inlineCallbacks
    def queue_pending_work(self):
        downloads, documents = yield self.download_database.get_pending_work(
            self.logger)
        for download in downloads:
            self.queue.put(
                functools.partial(
                    self.start_download,
                    download.file_number,
                    download.request_id,
                ))
        for document in documents:
            # TODO: self.logger doesn't include any info about the
            # DownloadStatus. When triggered from the "usual" path it has
            # ``file_number`` and ``request_id`` keys.
            self.queue.put(
                functools.partial(self.start_file_download, self.logger,
                                  document))

    @app.route("/")
    @instrumented_route
    def root(self, request):
        request.redirect("/efolder-express/")
        return succeed(None)

    @app.route("/efolder-express/")
    @instrumented_route
    def index(self, request):
        return self.render_template("index.html")

    @app.route("/efolder-express/download/", methods=["POST"])
    @instrumented_route
    @inlineCallbacks
    def download(self, request):
        file_number = request.args["file_number"][0]
        file_number = file_number.replace("-", "").replace(" ", "")

        request_id = str(uuid.uuid4())

        yield self.download_database.create_download(self.logger, request_id,
                                                     file_number)
        self.queue.put(
            functools.partial(self.start_download, file_number, request_id))

        request.redirect("/efolder-express/download/{}/".format(request_id))
        returnValue(None)

    @app.route("/efolder-express/download/<request_id>/")
    @instrumented_route
    @inlineCallbacks
    def download_status(self, request, request_id):
        download = yield self.download_database.get_download(
            self.logger, request_id=request_id)
        returnValue(self.render_template("download.html",
                                         {"status": download}))

    @app.route("/efolder-express/download/<request_id>/json/")
    @instrumented_route
    @inlineCallbacks
    def download_status_json(self, request, request_id):
        download = yield self.download_database.get_download(
            self.logger, request_id=request_id)
        html = self.render_template("_download_status.html", {
            "status": download,
        })
        request.setHeader("Content-Type", "application/json")
        returnValue(
            json.dumps({
                "completed": download.completed or download.state == "ERRORED",
                "html": html,
            }))

    @app.route("/efolder-express/download/<request_id>/zip/")
    @instrumented_route
    @inlineCallbacks
    def download_zip(self, request, request_id):
        download = yield self.download_database.get_download(
            self.logger, request_id=request_id)
        assert download.completed

        self.logger.bind(
            request_id=request_id,
            file_number=download.file_number,
        ).emit("download")

        document_types = yield self.document_types.wait()
        path = download.build_zip(self.jinja_env, self.fernet, document_types)

        request.setHeader(
            "Content-Disposition",
            "attachment; filename={}-eFolder.zip".format(download.file_number))

        resource = File(path, defaultType="application/zip")
        resource.isLeaf = True

        request.notifyFinish().addBoth(lambda *args, **kwargs: os.remove(path))

        returnValue(resource)
class DownloadEFolder(object):
    app = klein.Klein()

    def __init__(self, logger, download_database, storage_path, fernet,
                 vbms_client, queue, env_name):
        self.logger = logger
        self.download_database = download_database
        self.storage_path = storage_path
        self.fernet = fernet
        self.vbms_client = vbms_client
        self.queue = queue
        self.env_name = env_name

        self.jinja_env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(
                FilePath(__file__).parent().parent().child("templates").path,
            ),
            autoescape=True
        )
        self.document_types = DeferredValue()

    @classmethod
    def from_config(cls, reactor, logger, queue, config_path):
        with open(config_path) as f:
            config = yaml.safe_load(f)

        # TODO: bump this once alchimia properly handles pinning
        thread_pool = ThreadPool(minthreads=1, maxthreads=1)
        thread_pool.start()
        reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)

        return cls(
            logger,
            DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
            FilePath(config["storage"]["filesystem"]),
            fernet.MultiFernet([
                fernet.Fernet(key) for key in config["encryption_keys"]
            ]),
            VBMSClient(
                reactor,
                connect_vbms_path=config["connect_vbms"]["path"],
                bundle_path=config["connect_vbms"]["bundle_path"],
                endpoint_url=config["vbms"]["endpoint_url"],
                keyfile=config["vbms"]["keyfile"],
                samlfile=config["vbms"]["samlfile"],
                key=config["vbms"].get("key"),
                keypass=config["vbms"]["keypass"],
                ca_cert=config["vbms"].get("ca_cert"),
                client_cert=config["vbms"].get("client_cert"),
            ),
            queue,
            config["env"],
        )

    @classmethod
    def create_demo(cls, reactor, logger):
        return cls(
            logger=logger,
            download_database=DemoMemoryDownloadDatabase(),
            storage_path=None,
            fernet=None,
            vbms_client=None,
            queue=None,
            env_name="demo"
        )

    def render_template(self, template_name, data={}):
        t = self.jinja_env.get_template(template_name)
        return t.render(dict(data, env=self.env_name))

    @inlineCallbacks
    def start_download(self, file_number, request_id):
        logger = self.logger.bind(
            file_number=file_number, request_id=request_id
        )

        logger.emit("list_documents.start")
        try:
            documents = yield self.vbms_client.list_documents(
                logger, file_number
            )
        except VBMSError as e:
            logger.bind(
                stdout=e.stdout,
                stderr=e.stderr,
                exit_code=e.exit_code,
            ).emit("list_documents.error")
            yield self.download_database.mark_download_errored(
                logger, request_id
            )
        else:
            logger.emit("list_documents.success")

            documents = [
                Document.from_json(request_id, doc)
                for doc in documents
            ]
            yield self.download_database.create_documents(logger, documents)
            for doc in documents:
                self.queue.put(functools.partial(
                    self.start_file_download, logger, doc
                ))
            yield self.download_database.mark_download_manifest_downloaded(
                logger, request_id
            )

    @inlineCallbacks
    def start_file_download(self, logger, document):
        logger = logger.bind(document_id=document.document_id)
        logger.emit("get_document.start")

        try:
            contents = yield self.vbms_client.fetch_document_contents(
                logger, str(document.document_id)
            )
        except VBMSError as e:
            logger.bind(
                stdout=e.stdout,
                stderr=e.stderr,
                exit_code=e.exit_code,
            ).emit("get_document.error")
            yield self.download_database.mark_document_errored(
                logger, document
            )
        else:
            logger.emit("get_document.success")
            target = self.storage_path.child(str(uuid.uuid4()))
            target.setContent(self.fernet.encrypt(contents))
            yield self.download_database.set_document_content_location(
                logger, document, target.path
            )

    @inlineCallbacks
    def start_fetch_document_types(self):
        document_types = yield self.vbms_client.get_document_types(self.logger)
        self.document_types.completed({
            int(c["type_id"]): c["description"]
            for c in document_types
        })

    @inlineCallbacks
    def queue_pending_work(self):
        downloads, documents = yield self.download_database.get_pending_work(
            self.logger
        )
        for download in downloads:
            self.queue.put(functools.partial(
                self.start_download, download.file_number, download.request_id,
            ))
        for document in documents:
            # TODO: self.logger doesn't include any info about the
            # DownloadStatus. When triggered from the "usual" path it has
            # ``file_number`` and ``request_id`` keys.
            self.queue.put(functools.partial(
                self.start_file_download, self.logger, document
            ))

    @app.route("/")
    @instrumented_route
    def root(self, request):
        request.redirect("/efolder-express/")
        return succeed(None)

    @app.route("/efolder-express/")
    @instrumented_route
    def index(self, request):
        return self.render_template("index.html")

    @app.route("/efolder-express/download/", methods=["POST"])
    @instrumented_route
    @inlineCallbacks
    def download(self, request):
        file_number = request.args["file_number"][0]
        file_number = file_number.replace("-", "").replace(" ", "")

        request_id = str(uuid.uuid4())

        yield self.download_database.create_download(
            self.logger, request_id, file_number
        )
        self.queue.put(functools.partial(
            self.start_download, file_number, request_id
        ))

        request.redirect("/efolder-express/download/{}/".format(request_id))
        returnValue(None)

    @app.route("/efolder-express/download/<request_id>/")
    @instrumented_route
    @inlineCallbacks
    def download_status(self, request, request_id):
        download = yield self.download_database.get_download(
            self.logger, request_id=request_id
        )
        returnValue(self.render_template("download.html", {
            "status": download
        }))

    @app.route("/efolder-express/download/<request_id>/json/")
    @instrumented_route
    @inlineCallbacks
    def download_status_json(self, request, request_id):
        download = yield self.download_database.get_download(
            self.logger, request_id=request_id
        )
        html = self.render_template("_download_status.html", {
            "status": download,
        })
        request.setHeader("Content-Type", "application/json")
        returnValue(json.dumps({
            "completed": download.completed or download.state == "ERRORED",
            "html": html,
        }))

    @app.route("/efolder-express/download/<request_id>/zip/")
    @instrumented_route
    @inlineCallbacks
    def download_zip(self, request, request_id):
        download = yield self.download_database.get_download(
            self.logger, request_id=request_id
        )
        assert download.completed

        self.logger.bind(
            request_id=request_id,
            file_number=download.file_number,
        ).emit("download")

        document_types = yield self.document_types.wait()
        path = download.build_zip(self.jinja_env, self.fernet, document_types)

        request.setHeader(
            "Content-Disposition",
            "attachment; filename={}-eFolder.zip".format(download.file_number)
        )

        resource = File(path, defaultType="application/zip")
        resource.isLeaf = True

        request.notifyFinish().addBoth(lambda *args, **kwargs: os.remove(path))

        returnValue(resource)