def populate_export_download_task(export_instances,
filters,
download_id,
filename=None,
expiry=10 * 60 * 60):
export_file = get_export_file(
export_instances,
filters,
# We don't have a great way to calculate progress if it's a bulk download,
# so only track the progress for single instance exports.
progress_tracker=populate_export_download_task
if len(export_instances) == 1 else None)
file_format = Format.from_format(export_file.format)
filename = filename or export_instances[0].name
with export_file as file_:
db = get_blob_db()
db.put(file_, download_id, timeout=expiry)
expose_blob_download(
download_id,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header(filename,
file_format.extension),
download_id=download_id,
)
def dump_locations(domain, download_id, include_consumption, headers_only, task=None):
exporter = LocationExporter(domain, include_consumption=include_consumption,
headers_only=headers_only, async_task=task)
fd, path = tempfile.mkstemp()
writer = Excel2007ExportWriter()
writer.open(header_table=exporter.get_headers(), file=path)
with writer:
exporter.write_data(writer)
with open(path, 'rb') as file_:
db = get_blob_db()
expiry_mins = 60
db.put(
file_,
domain=domain,
parent_id=domain,
type_code=CODES.tempfile,
key=download_id,
timeout=expiry_mins,
)
file_format = Format.from_format(Excel2007ExportWriter.format)
expose_blob_download(
download_id,
expiry=expiry_mins * 60,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header('{}_locations'.format(domain), file_format.extension),
download_id=download_id,
)
def dump_locations(domain, download_id, include_consumption, headers_only, task=None):
exporter = LocationExporter(domain, include_consumption=include_consumption,
headers_only=headers_only, async_task=task)
fd, path = tempfile.mkstemp()
writer = Excel2007ExportWriter()
writer.open(header_table=exporter.get_headers(), file=path)
with writer:
exporter.write_data(writer)
with open(path, 'rb') as file_:
db = get_blob_db()
expiry_mins = 60
db.put(
file_,
domain=domain,
parent_id=domain,
type_code=CODES.tempfile,
key=download_id,
timeout=expiry_mins,
)
file_format = Format.from_format(Excel2007ExportWriter.format)
expose_blob_download(
download_id,
expiry=expiry_mins * 60,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header('{}_locations'.format(domain), file_format.extension),
download_id=download_id,
)
def populate_export_download_task(export_instances,
filters,
download_id,
filename=None,
expiry=10 * 60):
"""
:param expiry: Time period for the export to be available for download in minutes
"""
domain = export_instances[0].domain
with TransientTempfile() as temp_path, datadog_track_errors(
'populate_export_download_task'):
export_file = get_export_file(
export_instances,
filters,
temp_path,
# We don't have a great way to calculate progress if it's a bulk download,
# so only track the progress for single instance exports.
progress_tracker=populate_export_download_task
if len(export_instances) == 1 else None)
file_format = Format.from_format(export_file.format)
filename = filename or export_instances[0].name
with export_file as file_:
db = get_blob_db()
db.put(
file_,
domain=domain,
parent_id=domain,
type_code=CODES.data_export,
key=download_id,
timeout=expiry,
)
expose_blob_download(
download_id,
expiry=expiry * 60,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header(
filename, file_format.extension),
download_id=download_id,
)
email_requests = EmailExportWhenDoneRequest.objects.filter(
domain=domain, download_id=download_id)
for email_request in email_requests:
try:
couch_user = CouchUser.get_by_user_id(email_request.user_id,
domain=domain)
except CouchUser.AccountTypeError:
pass
else:
if couch_user is not None:
process_email_request(domain, download_id,
couch_user.get_email())
email_requests.delete()
def dump_locations(domain,
download_id,
include_consumption,
headers_only,
owner_id,
root_location_ids=None,
task=None,
**kwargs):
exporter = LocationExporter(domain,
include_consumption=include_consumption,
root_location_ids=root_location_ids,
headers_only=headers_only,
async_task=task,
**kwargs)
fd, path = tempfile.mkstemp()
writer = Excel2007ExportWriter()
writer.open(header_table=exporter.get_headers(), file=path)
with writer:
exporter.write_data(writer)
with open(path, 'rb') as file_:
db = get_blob_db()
expiry_mins = 60
db.put(
file_,
domain=domain,
parent_id=domain,
type_code=CODES.tempfile,
key=download_id,
timeout=expiry_mins,
)
file_format = Format.from_format(Excel2007ExportWriter.format)
filename = '{}_locations'.format(domain)
if len(root_location_ids) == 1:
root_location = SQLLocation.objects.get(
location_id=root_location_ids[0])
filename += '_{}'.format(root_location.name)
expose_blob_download(
download_id,
expiry=expiry_mins * 60,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header(filename,
file_format.extension),
download_id=download_id,
owner_ids=[owner_id],
)
def test_expose_blob_download(self):
content_disposition = 'text/xml'
download_id = 'abc123'
self.db.put(StringIO(u'content'), self.identifier)
expose_blob_download(self.identifier,
content_disposition=content_disposition,
download_id=download_id)
download = BlobDownload.get(download_id)
self.assertIsNotNone(download)
response = download.toHttpResponse()
self.assertEqual(next(response.streaming_content), u'content')
def _save_and_expose_zip(f, zip_name, domain, download_id):
expiry_minutes = 60
get_blob_db().put(
f,
key=download_id,
domain=domain,
parent_id=domain,
type_code=CODES.form_multimedia,
timeout=expiry_minutes,
)
expose_blob_download(
download_id,
expiry=expiry_minutes * 60, # seconds
mimetype='application/zip',
content_disposition=safe_filename_header(zip_name, 'zip'),
download_id=download_id,
)
def generate_toggle_csv_download(self, tag, download_id, username):
toggles = _get_toggles_with_tag(tag)
total = _get_toggle_item_count(toggles)
current_progress = [0]
def increment_progress():
current_progress[0] += 1
DownloadBase.set_progress(self, current_progress[0], total)
timeout_mins = 24 * 60
with TransientTempfile() as temp_path:
_write_toggle_data(temp_path, toggles, increment_progress)
with open(temp_path, 'rb') as file:
db = get_blob_db()
meta = db.put(
file,
domain="__system__",
parent_id="__system__",
type_code=CODES.tempfile,
key=download_id,
timeout=timeout_mins,
)
now = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
filename = f'{settings.SERVER_ENVIRONMENT}_toggle_export_{now}'
expose_blob_download(
download_id,
expiry=timeout_mins * 60,
content_disposition=safe_filename_header(filename, ".csv"),
download_id=download_id,
)
user = CouchUser.get_by_username(username)
if user:
url = absolute_reverse("retrieve_download", args=[download_id])
url += "?get_file"
valid_until = meta.expires_on.replace(
tzinfo=pytz.UTC).strftime(USER_DATETIME_FORMAT)
send_HTML_email("Feature Flag download ready",
user.get_email(),
html_content=inspect.cleandoc(f"""
Download URL: {url}
Download Valid until: {valid_until}
"""))
def test_expose_blob_download(self):
ref = expose_blob_download(
self.identifier,
expiry=60,
content_disposition='text/xml',
)
self.db.put(BytesIO(b'content'), meta=new_meta(key=ref.download_id))
response = BlobDownload.get(ref.download_id).toHttpResponse()
self.assertEqual(next(response.streaming_content), b'content')
def test_user_auth_required_access_denied(self):
ref = expose_blob_download('identifier',
expiry=60,
content_disposition='text/xml',
owner_ids=['foo'])
self.db.put(BytesIO(b'content'), meta=new_meta(key=ref.download_id))
response = self.client.get(
reverse('retrieve_download', args=[ref.download_id]) + "?get_file")
self.assertEqual(response.status_code, 403)
def populate_export_download_task(export_instances,
filters,
download_id,
filename=None,
expiry=10 * 60 * 60):
export_file = get_export_file(
export_instances,
filters,
# We don't have a great way to calculate progress if it's a bulk download,
# so only track the progress for single instance exports.
progress_tracker=populate_export_download_task
if len(export_instances) == 1 else None)
file_format = Format.from_format(export_file.format)
filename = filename or export_instances[0].name
with export_file as file_:
db = get_blob_db()
db.put(file_, download_id, timeout=expiry)
expose_blob_download(
download_id,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header(filename,
file_format.extension),
download_id=download_id,
)
domain = export_instances[0].domain
email_requests = EmailExportWhenDoneRequest.objects.filter(
domain=domain, download_id=download_id)
for email_request in email_requests:
try:
couch_user = CouchUser.get_by_user_id(email_request.user_id,
domain=domain)
except CouchUser.AccountTypeError:
pass
else:
if couch_user is not None:
process_email_request(domain, download_id,
couch_user.get_email())
email_requests.delete()
def save_dump_to_blob(self, temp_path):
with open(temp_path, 'rb') as file_:
blob_db = get_blob_db()
blob_db.put(file_, self.result_file_name,
timeout=60 * 48) # 48 hours
file_format = Format.from_format(Format.CSV)
file_name_header = safe_filename_header(self.result_file_name,
file_format.extension)
blob_dl_object = expose_blob_download(
self.result_file_name,
mimetype=file_format.mimetype,
content_disposition=file_name_header)
return blob_dl_object.download_id
def run_data_pull(data_pull_slug, domain, month, location_id=None, email=None):
subject = _('Custom ICDS Data Pull')
try:
filename = DataExporter(data_pull_slug,
"icds-ucr-citus",
month=month,
location_id=location_id).export()
except Exception:
if email:
message = _("""
Hi,
Could not generate the requested data pull.
The error has been notified. Please report as an issue for quicker followup
""")
send_html_email_async.delay(subject, [email],
message,
email_from=settings.DEFAULT_FROM_EMAIL)
raise
else:
if email and filename:
db = get_blob_db()
download_id = DownloadBase.new_id_prefix + make_uuid()
with open(filename, 'rb') as _file:
db.put(
_file,
domain=domain,
parent_id=domain,
type_code=CODES.data_export,
key=download_id,
timeout=24 * 60,
)
exposed_download = expose_blob_download(
filename,
expiry=24 * 60 * 60,
mimetype=Format.from_format(Format.ZIP).mimetype,
content_disposition=safe_filename_header(filename),
download_id=download_id)
os.remove(filename)
path = reverse(
'retrieve_download',
kwargs={'download_id': exposed_download.download_id})
link = f"{web.get_url_base()}{path}?get_file"
message = _("""
Hi,
Please download the data from {link}.
The data is available only for 24 hours.
""").format(link=link)
send_html_email_async.delay(subject, [email],
message,
email_from=settings.DEFAULT_FROM_EMAIL)
def populate_export_download_task(domain,
export_ids,
exports_type,
username,
es_filters,
download_id,
owner_id,
filename=None,
expiry=10 * 60):
"""
:param expiry: Time period for the export to be available for download in minutes
"""
email_requests = EmailExportWhenDoneRequest.objects.filter(
domain=domain, download_id=download_id)
if settings.STALE_EXPORT_THRESHOLD is not None and not email_requests.count(
):
delay = get_task_time_to_start(
populate_export_download_task.request.id)
if delay.total_seconds() > settings.STALE_EXPORT_THRESHOLD:
metrics_counter('commcare.exports.rejected_unfresh_export')
raise RejectedStaleExport()
export_instances = [
get_export(exports_type, domain, export_id, username)
for export_id in export_ids
]
with TransientTempfile() as temp_path, metrics_track_errors(
'populate_export_download_task'):
export_file = get_export_file(
export_instances,
es_filters,
temp_path,
# We don't have a great way to calculate progress if it's a bulk download,
# so only track the progress for single instance exports.
progress_tracker=populate_export_download_task
if len(export_instances) == 1 else None)
file_format = Format.from_format(export_file.format)
filename = filename or export_instances[0].name
with export_file as file_:
db = get_blob_db()
db.put(
file_,
domain=domain,
parent_id=domain,
type_code=CODES.data_export,
key=download_id,
timeout=expiry,
)
expose_blob_download(
download_id,
expiry=expiry * 60,
mimetype=file_format.mimetype,
content_disposition=safe_filename_header(
filename, file_format.extension),
download_id=download_id,
owner_ids=[owner_id],
)
for email_request in email_requests:
try:
couch_user = CouchUser.get_by_user_id(email_request.user_id,
domain=domain)
except CouchUser.AccountTypeError:
pass
else:
if couch_user is not None:
process_email_request(domain, download_id,
couch_user.get_email())
email_requests.delete()
