def test_delete_expired(self): q = Export.by_role_id(self.role_email.id) assert q.count() == 3, q.count() delete_expired_exports() q = Export.by_role_id(self.role_email.id) assert q.count() == 1, q.count() exp1 = Export.by_id(self.export1.id, deleted=False) assert exp1 is not None assert exp1.deleted is False exp2 = Export.by_id(self.export2.id, deleted=True) assert exp2 is not None assert exp2.deleted is True path = archive.load_file(self.export1.content_hash) assert path is not None assert path.exists() exp1.expires_at = datetime.utcnow() + timedelta(days=-1) db.session.add(exp1) db.session.commit() delete_expired_exports() q = Export.by_role_id(self.role_email.id) assert q.count() == 0, q.count() exp1 = Export.by_id(self.export1.id, deleted=True) assert exp1 is not None assert exp1.deleted is True path = archive.load_file(self.export1.content_hash) assert path is not None, path path = archive.load_file(self.export3.content_hash) assert path is None, path
def export_entities(export_id): export = Export.by_id(export_id) log.info("Export entities [%r]...", export) export_dir = ensure_path(mkdtemp(prefix="aleph.export.")) collections = {} try: filters = [export.meta.get("query", {"match_none": {}})] file_path = export_dir.joinpath("query-export.zip") with ZipFile(file_path, mode="w") as zf: excel_path = export_dir.joinpath(EXCEL_FILE) exporter = ExcelExporter(excel_path, extra=EXTRA_HEADERS) for entity in iter_proxies(filters=filters): collection_id = entity.context.get("collection_id") if collection_id not in collections: collections[collection_id] = get_collection(collection_id) collection = collections[collection_id] if collection is None: continue extra = [entity_url(entity.id), collection.get("label")] exporter.write(entity, extra=extra) write_document(export_dir, zf, collection, entity) if file_path.stat().st_size >= Export.MAX_FILE_SIZE: log.warn("Export too large: %r", export) break exporter.finalize() zf.write(excel_path, arcname=EXCEL_FILE) complete_export(export_id, file_path) except Exception: log.exception("Failed to process export [%s]", export_id) export = Export.by_id(export_id) export.set_status(status=Status.FAILED) db.session.commit() finally: shutil.rmtree(export_dir)
def export_matches(export_id): """Export the top N matches of cross-referencing for the given collection to an Excel formatted export.""" export = Export.by_id(export_id) export_dir = ensure_path(mkdtemp(prefix="aleph.export.")) try: role = Role.by_id(export.creator_id) authz = Authz.from_role(role) collection = Collection.by_id(export.collection_id) file_name = "%s - Crossreference.xlsx" % collection.label file_path = export_dir.joinpath(file_name) excel = ExcelWriter() headers = [ "Score", "Entity Name", "Entity Date", "Entity Countries", "Candidate Collection", "Candidate Name", "Candidate Date", "Candidate Countries", "Entity Link", "Candidate Link", ] sheet = excel.make_sheet("Cross-reference", headers) batch = [] for match in iter_matches(collection, authz): batch.append(match) if len(batch) >= BULK_PAGE: _iter_match_batch(excel, sheet, batch) batch = [] if len(batch): _iter_match_batch(excel, sheet, batch) with open(file_path, "wb") as fp: buffer = excel.get_bytesio() for data in buffer: fp.write(data) complete_export(export_id, file_path) except Exception: log.exception("Failed to process export [%s]", export_id) export = Export.by_id(export_id) export.set_status(status=Status.FAILED) db.session.commit() finally: shutil.rmtree(export_dir)
def complete_export(export_id, file_path): export = Export.by_id(export_id) file_path = ensure_path(file_path) export.file_name = safe_filename(file_path) export.file_size = file_path.stat().st_size export.content_hash = checksum(file_path) try: archive.archive_file(file_path, content_hash=export.content_hash, mime_type=export.mime_type) export.set_status(status=Status.SUCCESS) except Exception: log.exception("Failed to upload export: %s", export) export.set_status(status=Status.FAILED) db.session.commit() params = {"export": export} role = Role.by_id(export.creator_id) log.info("Export [%r] complete: %s", export, export.status) publish( Events.COMPLETE_EXPORT, params=params, channels=[role], ) send_export_notification(export)
def export_entities(export_id, result): from aleph.logic import resolver export_dir = ensure_path(mkdtemp(prefix="aleph.export.")) try: entities = [] stub = types.SimpleNamespace(result=result) for entity in result["results"]: resolver.queue(stub, Collection, entity.get("collection_id")) entities.append(model.get_proxy(entity)) resolver.resolve(stub) file_path = export_dir.joinpath("query-export.zip") zf = zipfile.ZipFile(file_path, "w") exporter = ExcelExporter(None, extra=EXTRA_HEADERS) for entity in entities: collection_id = entity.context.get("collection_id") collection = resolver.get(stub, Collection, collection_id) extra = [entity_url(entity.id), collection.get("label")] exporter.write(entity, extra=extra) write_document(export_dir, zf, collection, entity) content = exporter.get_bytesio().getvalue() zf.writestr("Export.xlsx", content) zf.close() complete_export(export_id, file_path) except Exception: log.exception("Failed to process export [%s]", export_id) export = Export.by_id(export_id) export.set_status(status=Export.STATUS_FAILED) db.session.commit() finally: shutil.rmtree(export_dir)
def export_entities(export_id): export = Export.by_id(export_id) log.info("Export entities [%r]...", export) export_dir = ensure_path(mkdtemp(prefix="aleph.export.")) collections = {} try: filters = [export.meta.get("query", {"match_none": {}})] file_path = export_dir.joinpath("export.zip") with ZipFile(file_path, mode="w") as zf: excel_name = safe_filename(export.label, extension="xlsx") excel_path = export_dir.joinpath(excel_name) exporter = ExcelExporter(excel_path, extra=EXTRA_HEADERS) for idx, entity in enumerate(iter_proxies(filters=filters)): collection_id = entity.context.get("collection_id") if collection_id not in collections: collections[collection_id] = get_collection(collection_id) collection = collections[collection_id] if collection is None: continue extra = [entity_url(entity.id), collection.get("label")] exporter.write(entity, extra=extra) write_document(export_dir, zf, collection, entity) if file_path.stat().st_size >= settings.EXPORT_MAX_SIZE: concern = "total size of the" zf.writestr("EXPORT_TOO_LARGE.txt", WARNING % concern) break if idx >= settings.EXPORT_MAX_RESULTS: concern = "number of" zf.writestr("EXPORT_TOO_LARGE.txt", WARNING % concern) break exporter.finalize() zf.write(excel_path, arcname=excel_name) file_name = "Export: %s" % export.label file_name = safe_filename(file_name, extension="zip") complete_export(export_id, file_path, file_name) except Exception: log.exception("Failed to process export [%s]", export_id) export = Export.by_id(export_id) export.set_status(status=Status.FAILED) db.session.commit() finally: shutil.rmtree(export_dir)
def download(export_id): """Downloads the exported file from the archive. --- get: summary: Download an export from the archive parameters: - description: export id in: path name: export_id required: true schema: type: string - description: Authorization token for an export in: query name: claim required: false schema: type: string description: A signed JWT with the object hash. responses: '200': description: OK content: '*/*': {} '404': description: Object does not exist. tags: - Export """ require(request.authz.logged_in) export = obj_or_404(Export.by_id(export_id, role_id=request.authz.id)) expires_after = export.expires_at - datetime.utcnow() url = archive.generate_publication_url( export.namespace, export.content_hash, mime_type=export.mime_type, expire=expires_after.total_seconds(), attachment_name=export.file_name, ) if url is not None: return redirect(url) local_path = archive.load_publication(export.namespace, export.content_hash) if local_path is None: raise NotFound() return send_file( str(local_path), as_attachment=True, conditional=True, attachment_filename=export.file_name, mimetype=export.mime_type, )
def get_export(export_id): if export_id is None: return key = cache.object_key(Export, export_id) data = cache.get_complex(key) if data is None: export = Export.by_id(export_id) if export is None: return log.debug("Export cache refresh: %r", export) data = export.to_dict() cache.set_complex(key, data, expires=cache.EXPIRE) return data
def complete_export(export_id, file_path=None): export = Export.by_id(export_id) if file_path: export.set_filepath(file_path) export.publish() db.session.commit() params = {"export": export} role = Role.by_id(export.creator_id) publish( Events.COMPLETE_EXPORT, params=params, channels=[role], ) send_export_notification(export)
def get_export(export_id): if export_id is None: return export = Export.by_id(export_id, deleted=True) if export is not None: return export.to_dict()