def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60): export_file = get_export_file( export_instances, filters, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put(file_, download_id, timeout=expiry) expose_blob_download( download_id, mimetype=file_format.mimetype, content_disposition=safe_filename_header(filename, file_format.extension), download_id=download_id, )
def dump_locations(domain, download_id, include_consumption, headers_only, task=None): exporter = LocationExporter(domain, include_consumption=include_consumption, headers_only=headers_only, async_task=task) fd, path = tempfile.mkstemp() writer = Excel2007ExportWriter() writer.open(header_table=exporter.get_headers(), file=path) with writer: exporter.write_data(writer) with open(path, 'rb') as file_: db = get_blob_db() expiry_mins = 60 db.put( file_, domain=domain, parent_id=domain, type_code=CODES.tempfile, key=download_id, timeout=expiry_mins, ) file_format = Format.from_format(Excel2007ExportWriter.format) expose_blob_download( download_id, expiry=expiry_mins * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header('{}_locations'.format(domain), file_format.extension), download_id=download_id, )
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60): """ :param expiry: Time period for the export to be available for download in minutes """ domain = export_instances[0].domain with TransientTempfile() as temp_path, datadog_track_errors( 'populate_export_download_task'): export_file = get_export_file( export_instances, filters, temp_path, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put( file_, domain=domain, parent_id=domain, type_code=CODES.data_export, key=download_id, timeout=expiry, ) expose_blob_download( download_id, expiry=expiry * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header( filename, file_format.extension), download_id=download_id, ) email_requests = EmailExportWhenDoneRequest.objects.filter( domain=domain, download_id=download_id) for email_request in email_requests: try: couch_user = CouchUser.get_by_user_id(email_request.user_id, domain=domain) except CouchUser.AccountTypeError: pass else: if couch_user is not None: process_email_request(domain, download_id, couch_user.get_email()) email_requests.delete()
def dump_locations(domain, download_id, include_consumption, headers_only, owner_id, root_location_ids=None, task=None, **kwargs): exporter = LocationExporter(domain, include_consumption=include_consumption, root_location_ids=root_location_ids, headers_only=headers_only, async_task=task, **kwargs) fd, path = tempfile.mkstemp() writer = Excel2007ExportWriter() writer.open(header_table=exporter.get_headers(), file=path) with writer: exporter.write_data(writer) with open(path, 'rb') as file_: db = get_blob_db() expiry_mins = 60 db.put( file_, domain=domain, parent_id=domain, type_code=CODES.tempfile, key=download_id, timeout=expiry_mins, ) file_format = Format.from_format(Excel2007ExportWriter.format) filename = '{}_locations'.format(domain) if len(root_location_ids) == 1: root_location = SQLLocation.objects.get( location_id=root_location_ids[0]) filename += '_{}'.format(root_location.name) expose_blob_download( download_id, expiry=expiry_mins * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header(filename, file_format.extension), download_id=download_id, owner_ids=[owner_id], )
def test_expose_blob_download(self): content_disposition = 'text/xml' download_id = 'abc123' self.db.put(StringIO(u'content'), self.identifier) expose_blob_download(self.identifier, content_disposition=content_disposition, download_id=download_id) download = BlobDownload.get(download_id) self.assertIsNotNone(download) response = download.toHttpResponse() self.assertEqual(next(response.streaming_content), u'content')
def _save_and_expose_zip(f, zip_name, domain, download_id): expiry_minutes = 60 get_blob_db().put( f, key=download_id, domain=domain, parent_id=domain, type_code=CODES.form_multimedia, timeout=expiry_minutes, ) expose_blob_download( download_id, expiry=expiry_minutes * 60, # seconds mimetype='application/zip', content_disposition=safe_filename_header(zip_name, 'zip'), download_id=download_id, )
def generate_toggle_csv_download(self, tag, download_id, username): toggles = _get_toggles_with_tag(tag) total = _get_toggle_item_count(toggles) current_progress = [0] def increment_progress(): current_progress[0] += 1 DownloadBase.set_progress(self, current_progress[0], total) timeout_mins = 24 * 60 with TransientTempfile() as temp_path: _write_toggle_data(temp_path, toggles, increment_progress) with open(temp_path, 'rb') as file: db = get_blob_db() meta = db.put( file, domain="__system__", parent_id="__system__", type_code=CODES.tempfile, key=download_id, timeout=timeout_mins, ) now = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") filename = f'{settings.SERVER_ENVIRONMENT}_toggle_export_{now}' expose_blob_download( download_id, expiry=timeout_mins * 60, content_disposition=safe_filename_header(filename, ".csv"), download_id=download_id, ) user = CouchUser.get_by_username(username) if user: url = absolute_reverse("retrieve_download", args=[download_id]) url += "?get_file" valid_until = meta.expires_on.replace( tzinfo=pytz.UTC).strftime(USER_DATETIME_FORMAT) send_HTML_email("Feature Flag download ready", user.get_email(), html_content=inspect.cleandoc(f""" Download URL: {url} Download Valid until: {valid_until} """))
def test_expose_blob_download(self): ref = expose_blob_download( self.identifier, expiry=60, content_disposition='text/xml', ) self.db.put(BytesIO(b'content'), meta=new_meta(key=ref.download_id)) response = BlobDownload.get(ref.download_id).toHttpResponse() self.assertEqual(next(response.streaming_content), b'content')
def test_user_auth_required_access_denied(self): ref = expose_blob_download('identifier', expiry=60, content_disposition='text/xml', owner_ids=['foo']) self.db.put(BytesIO(b'content'), meta=new_meta(key=ref.download_id)) response = self.client.get( reverse('retrieve_download', args=[ref.download_id]) + "?get_file") self.assertEqual(response.status_code, 403)
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60): export_file = get_export_file( export_instances, filters, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put(file_, download_id, timeout=expiry) expose_blob_download( download_id, mimetype=file_format.mimetype, content_disposition=safe_filename_header(filename, file_format.extension), download_id=download_id, ) domain = export_instances[0].domain email_requests = EmailExportWhenDoneRequest.objects.filter( domain=domain, download_id=download_id) for email_request in email_requests: try: couch_user = CouchUser.get_by_user_id(email_request.user_id, domain=domain) except CouchUser.AccountTypeError: pass else: if couch_user is not None: process_email_request(domain, download_id, couch_user.get_email()) email_requests.delete()
def save_dump_to_blob(self, temp_path): with open(temp_path, 'rb') as file_: blob_db = get_blob_db() blob_db.put(file_, self.result_file_name, timeout=60 * 48) # 48 hours file_format = Format.from_format(Format.CSV) file_name_header = safe_filename_header(self.result_file_name, file_format.extension) blob_dl_object = expose_blob_download( self.result_file_name, mimetype=file_format.mimetype, content_disposition=file_name_header) return blob_dl_object.download_id
def run_data_pull(data_pull_slug, domain, month, location_id=None, email=None): subject = _('Custom ICDS Data Pull') try: filename = DataExporter(data_pull_slug, "icds-ucr-citus", month=month, location_id=location_id).export() except Exception: if email: message = _(""" Hi, Could not generate the requested data pull. The error has been notified. Please report as an issue for quicker followup """) send_html_email_async.delay(subject, [email], message, email_from=settings.DEFAULT_FROM_EMAIL) raise else: if email and filename: db = get_blob_db() download_id = DownloadBase.new_id_prefix + make_uuid() with open(filename, 'rb') as _file: db.put( _file, domain=domain, parent_id=domain, type_code=CODES.data_export, key=download_id, timeout=24 * 60, ) exposed_download = expose_blob_download( filename, expiry=24 * 60 * 60, mimetype=Format.from_format(Format.ZIP).mimetype, content_disposition=safe_filename_header(filename), download_id=download_id) os.remove(filename) path = reverse( 'retrieve_download', kwargs={'download_id': exposed_download.download_id}) link = f"{web.get_url_base()}{path}?get_file" message = _(""" Hi, Please download the data from {link}. The data is available only for 24 hours. """).format(link=link) send_html_email_async.delay(subject, [email], message, email_from=settings.DEFAULT_FROM_EMAIL)
def populate_export_download_task(domain, export_ids, exports_type, username, es_filters, download_id, owner_id, filename=None, expiry=10 * 60): """ :param expiry: Time period for the export to be available for download in minutes """ email_requests = EmailExportWhenDoneRequest.objects.filter( domain=domain, download_id=download_id) if settings.STALE_EXPORT_THRESHOLD is not None and not email_requests.count( ): delay = get_task_time_to_start( populate_export_download_task.request.id) if delay.total_seconds() > settings.STALE_EXPORT_THRESHOLD: metrics_counter('commcare.exports.rejected_unfresh_export') raise RejectedStaleExport() export_instances = [ get_export(exports_type, domain, export_id, username) for export_id in export_ids ] with TransientTempfile() as temp_path, metrics_track_errors( 'populate_export_download_task'): export_file = get_export_file( export_instances, es_filters, temp_path, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put( file_, domain=domain, parent_id=domain, type_code=CODES.data_export, key=download_id, timeout=expiry, ) expose_blob_download( download_id, expiry=expiry * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header( filename, file_format.extension), download_id=download_id, owner_ids=[owner_id], ) for email_request in email_requests: try: couch_user = CouchUser.get_by_user_id(email_request.user_id, domain=domain) except CouchUser.AccountTypeError: pass else: if couch_user is not None: process_email_request(domain, download_id, couch_user.get_email()) email_requests.delete()