def _pull_resource(self, request): resource_slug = self.pull_resource_form.cleaned_data['resource_slug'] project_slug = self.pull_resource_form.cleaned_data['transifex_project_slug'] file_response = self._generate_response_file(request.domain, project_slug, resource_slug) if isinstance(file_response, Workbook): content = get_file_content_from_workbook(file_response) response = HttpResponse(content, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header(resource_slug, "xlsx") else: response = HttpResponse(file_response, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header(project_slug, "zip") return response
def dump_locations(domain, download_id, include_consumption, headers_only, task=None): exporter = LocationExporter(domain, include_consumption=include_consumption, headers_only=headers_only, async_task=task) fd, path = tempfile.mkstemp() writer = Excel2007ExportWriter() writer.open(header_table=exporter.get_headers(), file=path) with writer: exporter.write_data(writer) with open(path, 'rb') as file_: db = get_blob_db() expiry_mins = 60 db.put( file_, domain=domain, parent_id=domain, type_code=CODES.tempfile, key=download_id, timeout=expiry_mins, ) file_format = Format.from_format(Excel2007ExportWriter.format) expose_blob_download( download_id, expiry=expiry_mins * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header('{}_locations'.format(domain), file_format.extension), download_id=download_id, )
def _zip_file_response(self): uploaded_file = self.convert_translation_form.cleaned_data.get( 'upload_file') uploaded_zipfile = ZipFile(uploaded_file) mem_file = BytesIO() with ZipFile(mem_file, 'w') as zipfile: for file_info in uploaded_zipfile.filelist: filename = file_info.filename if filename.endswith('.po'): po_file = BytesIO(uploaded_zipfile.read(filename)) wb = self._generate_excel_file(po_file) result_filename = filename.split('.po')[0] zipfile.writestr(result_filename + '.xlsx', get_file_content_from_workbook(wb)) elif filename.endswith('.xls') or filename.endswith('.xlsx'): worksheet = openpyxl.load_workbook( BytesIO(uploaded_zipfile.read(filename))).worksheets[0] po_file_content = self._generate_po_content(worksheet) result_filename = filename.split('.xls')[0] zipfile.writestr(result_filename + '.po', po_file_content) else: assert False, "unexpected filename: {}".format(filename) mem_file.seek(0) response = HttpResponse(mem_file, content_type="text/html") zip_filename = 'Converted-' + uploaded_zipfile.filename.split( '.zip')[0] response['Content-Disposition'] = safe_filename_header( zip_filename, "zip") return response
def _po_file_response(self): uploaded_file = self.convert_translation_form.cleaned_data.get('upload_file') worksheet = openpyxl.load_workbook(uploaded_file).worksheets[0] content = self._generate_po_content(worksheet) response = HttpResponse(content, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header(worksheet.title, 'po') return response
def export_response(file, format, filename, checkpoint=None): """ Get an http response for an export file can be either a io.BytesIO or io.StringIO or an open file object (which this function is responsible for closing) """ from couchexport.export import Format if not filename: filename = "NAMELESS EXPORT" format = Format.from_format(format) if isinstance(file, TempBase): file = file.file if isinstance(file, (io.BytesIO, io.StringIO)): response = HttpResponse(file.getvalue(), content_type=format.mimetype) # I don't know why we need to close the file. Keeping around. file.close() else: response = StreamingHttpResponse(FileWrapper(file), content_type=format.mimetype) if format.download: from corehq.util.files import safe_filename_header response['Content-Disposition'] = safe_filename_header( filename, format.extension) if checkpoint: response['X-CommCareHQ-Export-Token'] = checkpoint.get_id return response
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60): export_file = get_export_file( export_instances, filters, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name payload = export_file.file.payload expose_cached_download( payload, expiry, ".{}".format(file_format.extension), mimetype=file_format.mimetype, content_disposition=safe_filename_header(filename, file_format.extension), download_id=download_id, ) export_file.file.delete()
def export_response(file, format, filename, checkpoint=None): """ Get an http response for an export file can be either a io.BytesIO or io.StringIO or an open file object (which this function is responsible for closing) """ from couchexport.export import Format if not filename: filename = "NAMELESS EXPORT" format = Format.from_format(format) if isinstance(file, TempBase): file = file.file if isinstance(file, (io.BytesIO, io.StringIO)): response = HttpResponse(file.getvalue(), content_type=format.mimetype) # I don't know why we need to close the file. Keeping around. file.close() else: response = StreamingHttpResponse(FileWrapper(file), content_type=format.mimetype) if format.download: from corehq.util.files import safe_filename_header response['Content-Disposition'] = safe_filename_header(filename, format.extension) if checkpoint: response['X-CommCareHQ-Export-Token'] = checkpoint.get_id return response
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60): export_file = get_export_file( export_instances, filters, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put(file_, download_id, timeout=expiry) expose_blob_download( download_id, mimetype=file_format.mimetype, content_disposition=safe_filename_header(filename, file_format.extension), download_id=download_id, )
def _excel_file_response(self): wb = self._generate_excel_file() content = get_file_content_from_workbook(wb) response = HttpResponse(content, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header( self._uploaded_file_name.split('.po')[0], 'xlsx') return response
def _generate_summary_response(self, transitions, uploaded_filename): filename = uploaded_filename.split('.')[0] + " Summary" response_file = Dumper(self.domain).dump(transitions) response = HttpResponse(response_file, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header( filename, 'xlsx') return response
def get(self, request, *args, **kwargs): file_id = self.kwargs.get('file_id', None) content_type = Format.from_format('xlsx') response = HttpResponse(get_file_from_blobdb(file_id).read(), content_type=content_type.mimetype) response['Content-Disposition'] = safe_filename_header( 'unified_beneficiary_list', content_type.extension) return response
def _pull_resource(self, request): resource_slug = self.pull_resource_form.cleaned_data['resource_slug'] wb = self._generate_excel_file(request.domain, resource_slug) content = get_file_content_from_workbook(wb) response = HttpResponse(content, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header( resource_slug, 'xlsx') return response
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60): """ :param expiry: Time period for the export to be available for download in minutes """ domain = export_instances[0].domain with TransientTempfile() as temp_path, datadog_track_errors( 'populate_export_download_task'): export_file = get_export_file( export_instances, filters, temp_path, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put( file_, domain=domain, parent_id=domain, type_code=CODES.data_export, key=download_id, timeout=expiry, ) expose_blob_download( download_id, expiry=expiry * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header( filename, file_format.extension), download_id=download_id, ) email_requests = EmailExportWhenDoneRequest.objects.filter( domain=domain, download_id=download_id) for email_request in email_requests: try: couch_user = CouchUser.get_by_user_id(email_request.user_id, domain=domain) except CouchUser.AccountTypeError: pass else: if couch_user is not None: process_email_request(domain, download_id, couch_user.get_email()) email_requests.delete()
def get(self, request, *args, **kwargs): file_id = self.kwargs.get('file_id', None) content_type = Format.from_format('xlsx') response = HttpResponse( get_file_from_blobdb(file_id).read(), content_type=content_type.mimetype ) response['Content-Disposition'] = safe_filename_header( 'unified_beneficiary_list', content_type.extension ) return response
def save_dump_to_blob(data_file_path, data_file_name, result_file_format): with open(data_file_path, 'rb') as file_: blob_db = get_blob_db() blob_db.put(file_, data_file_name, timeout=60 * 24) # 24 hours file_format = Format.from_format(result_file_format) file_name_header = safe_filename_header(data_file_name, file_format.extension) blob_dl_object = expose_blob_download( data_file_name, mimetype=file_format.mimetype, content_disposition=file_name_header) return blob_dl_object.download_id
def save_dump_to_blob(self, temp_path): with open(temp_path, 'rb') as file_: blob_db = get_blob_db() blob_db.put(file_, self.result_file_name, timeout=60 * 48) # 48 hours file_format = Format.from_format(Format.CSV) file_name_header = safe_filename_header(self.result_file_name, file_format.extension) blob_dl_object = expose_blob_download( self.result_file_name, mimetype=file_format.mimetype, content_disposition=file_name_header) return blob_dl_object.download_id
def get_download_response(payload, content_length, content_format, filename, request=None): """ :param payload: File like object. :param content_length: Size of payload in bytes :param content_format: ``couchexport.models.Format`` instance :param filename: Name of the download :param request: The request. Used to determine if a range response should be given. :return: HTTP response """ ranges = None if request and "HTTP_RANGE" in request.META: try: ranges = parse_range_header(request.META['HTTP_RANGE'], content_length) except ValueError: pass if ranges and len(ranges.ranges) != 1: ranges = None response = StreamingHttpResponse(content_type=content_format.mimetype) if content_format.download: response['Content-Disposition'] = safe_filename_header(filename) response["Content-Length"] = content_length response["Accept-Ranges"] = "bytes" if ranges: start, stop = ranges.ranges[0] if stop is not None and stop > content_length: # requested range not satisfiable return HttpResponse(status=416) response.streaming_content = RangedFileWrapper(payload, start=start, stop=stop or float("inf")) end = stop or content_length response["Content-Range"] = "bytes %d-%d/%d" % (start, end - 1, content_length) response["Content-Length"] = end - start response.status_code = 206 else: response.streaming_content = FileWrapper(payload) return response
def run_data_pull(data_pull_slug, domain, month, location_id=None, email=None): subject = _('Custom ICDS Data Pull') try: filename = DataExporter(data_pull_slug, "icds-ucr-citus", month=month, location_id=location_id).export() except Exception: if email: message = _(""" Hi, Could not generate the requested data pull. The error has been notified. Please report as an issue for quicker followup """) send_html_email_async.delay(subject, [email], message, email_from=settings.DEFAULT_FROM_EMAIL) raise else: if email and filename: db = get_blob_db() download_id = DownloadBase.new_id_prefix + make_uuid() with open(filename, 'rb') as _file: db.put( _file, domain=domain, parent_id=domain, type_code=CODES.data_export, key=download_id, timeout=24 * 60, ) exposed_download = expose_blob_download( filename, expiry=24 * 60 * 60, mimetype=Format.from_format(Format.ZIP).mimetype, content_disposition=safe_filename_header(filename), download_id=download_id) os.remove(filename) path = reverse( 'retrieve_download', kwargs={'download_id': exposed_download.download_id}) link = f"{web.get_url_base()}{path}?get_file" message = _(""" Hi, Please download the data from {link}. The data is available only for 24 hours. """).format(link=link) send_html_email_async.delay(subject, [email], message, email_from=settings.DEFAULT_FROM_EMAIL)
def download_location_reassignment_template(request, domain): location_id = request.GET.get('location_id') if not location_id or not user_can_access_location_id(domain, request.couch_user, location_id): messages.error(request, _("Please select a location.")) return HttpResponseRedirect(reverse(LocationReassignmentView.urlname, args=[domain])) location = SQLLocation.active_objects.get(location_id=location_id, domain=domain) response_file = DownloadUsers(location).dump() response = HttpResponse(response_file, content_type="text/html; charset=utf-8") timezone = get_timezone_for_user(request.couch_user, domain) creation_time = datetime.now(timezone).strftime(FILENAME_DATETIME_FORMAT) filename = f"[{domain}] {location.name} Location Reassignment Request Template {creation_time}" response['Content-Disposition'] = safe_filename_header(filename, 'xlsx') return response
def dump_locations(domain, download_id, include_consumption, headers_only, owner_id, root_location_ids=None, task=None, **kwargs): exporter = LocationExporter(domain, include_consumption=include_consumption, root_location_ids=root_location_ids, headers_only=headers_only, async_task=task, **kwargs) fd, path = tempfile.mkstemp() writer = Excel2007ExportWriter() writer.open(header_table=exporter.get_headers(), file=path) with writer: exporter.write_data(writer) with open(path, 'rb') as file_: db = get_blob_db() expiry_mins = 60 db.put( file_, domain=domain, parent_id=domain, type_code=CODES.tempfile, key=download_id, timeout=expiry_mins, ) file_format = Format.from_format(Excel2007ExportWriter.format) filename = '{}_locations'.format(domain) if len(root_location_ids) == 1: root_location = SQLLocation.objects.get( location_id=root_location_ids[0]) filename += '_{}'.format(root_location.name) expose_blob_download( download_id, expiry=expiry_mins * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header(filename, file_format.extension), download_id=download_id, owner_ids=[owner_id], )
def _po_file_response(self): uploaded_file = self.convert_translation_form.cleaned_data.get( 'upload_file') worksheet = openpyxl.load_workbook(uploaded_file).worksheets[0] po_file_generator = self._generate_po_file(worksheet) try: with open(po_file_generator.generated_files[0][1], 'r', encoding="utf-8") as f: content = f.read() finally: po_file_generator.cleanup() response = HttpResponse(content, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header( worksheet.title, 'po') return response
def _save_and_expose_zip(f, zip_name, domain, download_id): expiry_minutes = 60 get_blob_db().put( f, key=download_id, domain=domain, parent_id=domain, type_code=CODES.form_multimedia, timeout=expiry_minutes, ) expose_blob_download( download_id, expiry=expiry_minutes * 60, # seconds mimetype='application/zip', content_disposition=safe_filename_header(zip_name, 'zip'), download_id=download_id, )
def generate_toggle_csv_download(self, tag, download_id, username): toggles = _get_toggles_with_tag(tag) total = _get_toggle_item_count(toggles) current_progress = [0] def increment_progress(): current_progress[0] += 1 DownloadBase.set_progress(self, current_progress[0], total) timeout_mins = 24 * 60 with TransientTempfile() as temp_path: _write_toggle_data(temp_path, toggles, increment_progress) with open(temp_path, 'rb') as file: db = get_blob_db() meta = db.put( file, domain="__system__", parent_id="__system__", type_code=CODES.tempfile, key=download_id, timeout=timeout_mins, ) now = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") filename = f'{settings.SERVER_ENVIRONMENT}_toggle_export_{now}' expose_blob_download( download_id, expiry=timeout_mins * 60, content_disposition=safe_filename_header(filename, ".csv"), download_id=download_id, ) user = CouchUser.get_by_username(username) if user: url = absolute_reverse("retrieve_download", args=[download_id]) url += "?get_file" valid_until = meta.expires_on.replace( tzinfo=pytz.UTC).strftime(USER_DATETIME_FORMAT) send_HTML_email("Feature Flag download ready", user.get_email(), html_content=inspect.cleandoc(f""" Download URL: {url} Download Valid until: {valid_until} """))
def expose_zipped_blob_download(data_path, filename, format, domain): """Expose zipped file content as a blob download :param data_path: Path to data file. Will be deleted. :param filename: File name. :param format: `couchexport.models.Format` constant. :param domain: Domain name. :returns: A link to download the file. """ try: _, zip_temp_path = tempfile.mkstemp(".zip") with ZipFile(zip_temp_path, 'w') as zip_file_: zip_file_.write(data_path, filename) finally: os.remove(data_path) try: expiry_mins = 60 * 24 file_format = Format.from_format(format) file_name_header = safe_filename_header(filename, file_format.extension) ref = expose_blob_download( filename, expiry=expiry_mins * 60, mimetype=file_format.mimetype, content_disposition=file_name_header ) with open(zip_temp_path, 'rb') as file_: get_blob_db().put( file_, domain=domain, parent_id=domain, type_code=CODES.tempfile, key=ref.download_id, timeout=expiry_mins ) finally: os.remove(zip_temp_path) return "%s%s?%s" % ( get_url_base(), reverse('retrieve_download', kwargs={'download_id': ref.download_id}), "get_file" # download immediately rather than rendering page )
def populate_export_download_task(export_instances, filters, download_id, filename=None, expiry=10 * 60 * 60): export_file = get_export_file( export_instances, filters, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put(file_, download_id, timeout=expiry) expose_blob_download( download_id, mimetype=file_format.mimetype, content_disposition=safe_filename_header(filename, file_format.extension), download_id=download_id, ) domain = export_instances[0].domain email_requests = EmailExportWhenDoneRequest.objects.filter( domain=domain, download_id=download_id) for email_request in email_requests: try: couch_user = CouchUser.get_by_user_id(email_request.user_id, domain=domain) except CouchUser.AccountTypeError: pass else: if couch_user is not None: process_email_request(domain, download_id, couch_user.get_email()) email_requests.delete()
def _zip_file_response(self): uploaded_file = self.convert_translation_form.cleaned_data.get('upload_file') uploaded_zipfile = ZipFile(uploaded_file) mem_file = BytesIO() with ZipFile(mem_file, 'w') as zipfile: for file_info in uploaded_zipfile.filelist: filename = file_info.filename if filename.endswith('.po'): po_file = BytesIO(uploaded_zipfile.read(filename)) wb = self._generate_excel_file(po_file) result_filename = filename.split('.po')[0] zipfile.writestr(result_filename + '.xlsx', get_file_content_from_workbook(wb)) elif filename.endswith('.xls') or filename.endswith('.xlsx'): worksheet = openpyxl.load_workbook(BytesIO(uploaded_zipfile.read(filename))).worksheets[0] po_file_content = self._generate_po_content(worksheet) result_filename = filename.split('.xls')[0] zipfile.writestr(result_filename + '.po', po_file_content) else: assert False, "unexpected filename: {}".format(filename) mem_file.seek(0) response = HttpResponse(mem_file, content_type="text/html") zip_filename = 'Converted-' + uploaded_zipfile.filename.split('.zip')[0] response['Content-Disposition'] = safe_filename_header(zip_filename, "zip") return response
def populate_export_download_task(domain, export_ids, exports_type, username, es_filters, download_id, owner_id, filename=None, expiry=10 * 60): """ :param expiry: Time period for the export to be available for download in minutes """ email_requests = EmailExportWhenDoneRequest.objects.filter( domain=domain, download_id=download_id) if settings.STALE_EXPORT_THRESHOLD is not None and not email_requests.count( ): delay = get_task_time_to_start( populate_export_download_task.request.id) if delay.total_seconds() > settings.STALE_EXPORT_THRESHOLD: metrics_counter('commcare.exports.rejected_unfresh_export') raise RejectedStaleExport() export_instances = [ get_export(exports_type, domain, export_id, username) for export_id in export_ids ] with TransientTempfile() as temp_path, metrics_track_errors( 'populate_export_download_task'): export_file = get_export_file( export_instances, es_filters, temp_path, # We don't have a great way to calculate progress if it's a bulk download, # so only track the progress for single instance exports. progress_tracker=populate_export_download_task if len(export_instances) == 1 else None) file_format = Format.from_format(export_file.format) filename = filename or export_instances[0].name with export_file as file_: db = get_blob_db() db.put( file_, domain=domain, parent_id=domain, type_code=CODES.data_export, key=download_id, timeout=expiry, ) expose_blob_download( download_id, expiry=expiry * 60, mimetype=file_format.mimetype, content_disposition=safe_filename_header( filename, file_format.extension), download_id=download_id, owner_ids=[owner_id], ) for email_request in email_requests: try: couch_user = CouchUser.get_by_user_id(email_request.user_id, domain=domain) except CouchUser.AccountTypeError: pass else: if couch_user is not None: process_email_request(domain, download_id, couch_user.get_email()) email_requests.delete()
def test_format_and_set_as_header(self, filename, expected_filename): self.assertEqual(safe_filename(filename), expected_filename) self.assertWorksAsHeader(safe_filename_header(filename))
def test_header_format(self): header = safe_filename_header('test', 'zip') self.assertWorksAsHeader(header) expected = 'attachment; filename="test.zip"; filename*=UTF-8\'\'test.zip' self.assertEqual(header, expected)
def _excel_file_response(self): wb = self._generate_excel_file(self.convert_translation_form.cleaned_data.get('upload_file')) content = get_file_content_from_workbook(wb) response = HttpResponse(content, content_type="text/html; charset=utf-8") response['Content-Disposition'] = safe_filename_header(self._uploaded_file_name.split('.po')[0], 'xlsx') return response