def bulk_import_async(config, domain, excel_id):
case_upload = CaseUpload.get(excel_id)
try:
case_upload.check_file()
except ImporterError as e:
update_task_state(bulk_import_async, states.FAILURE,
{'errors': get_importer_error_message(e)})
raise Ignore()
try:
with case_upload.get_spreadsheet() as spreadsheet:
result = do_import(spreadsheet,
config,
domain,
task=bulk_import_async,
record_form_callback=case_upload.record_form)
_alert_on_result(result, domain)
# return compatible with soil
return {'messages': result}
except ImporterError as e:
update_task_state(bulk_import_async, states.FAILURE,
{'errors': get_importer_error_message(e)})
raise Ignore()
finally:
store_task_result.delay(excel_id)
def exit_celery_with_error_message(task, error_message):
"""
Call this function and return the value from within a celery task to abort
with an error message that gets passed on in a way that case importer
will pick up and display.
Currently it doesn't return anything and does all its magic by manually
setting task metadata and raising Ignore,
but the internals could change to do this through a return value instead.
"""
update_task_state(task, states.FAILURE, get_interned_exception(error_message))
raise Ignore()
def bulk_import_async(config, domain, excel_id):
case_upload = CaseUpload.get(excel_id)
try:
case_upload.check_file()
except ImporterError as e:
update_task_state(bulk_import_async, states.FAILURE, {'errors': get_importer_error_message(e)})
raise Ignore()
try:
with case_upload.get_spreadsheet() as spreadsheet:
result = do_import(spreadsheet, config, domain, task=bulk_import_async,
record_form_callback=case_upload.record_form)
_alert_on_result(result, domain)
# return compatible with soil
return {
'messages': result
}
except ImporterError as e:
update_task_state(bulk_import_async, states.FAILURE, {'errors': get_importer_error_message(e)})
raise Ignore()
finally:
store_task_result.delay(excel_id)
def build_application_zip(include_multimedia_files,
include_index_files,
app,
download_id,
build_profile_id=None,
compress_zip=False,
filename="commcare.zip",
download_targeted_version=False):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
initial_progress = 10 # early on indicate something is happening
file_progress = 50.0 # arbitrarily say building files takes half the total time
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(
settings.SHARED_DRIVE_CONF.transfer_dir,
"{}{}{}{}{}".format(app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version, build_profile_id))
if download_targeted_version:
fpath += '-targeted'
else:
dummy, fpath = tempfile.mkstemp()
DownloadBase.set_progress(build_application_zip, initial_progress, 100)
if not (os.path.isfile(fpath)
and use_transfer): # Don't rebuild the file if it is already there
files, errors, file_count = iter_app_files(
app,
include_multimedia_files,
include_index_files,
build_profile_id,
download_targeted_version=download_targeted_version,
)
if toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain):
manifest = json.dumps(
{
'include_multimedia_files': include_multimedia_files,
'include_index_files': include_index_files,
'download_id': download_id,
'build_profile_id': build_profile_id,
'compress_zip': compress_zip,
'filename': filename,
'download_targeted_version': download_targeted_version,
'app': app.to_json(),
},
indent=4)
files = itertools.chain(files, [('manifest.json', manifest)])
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
progress = initial_progress
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
progress += file_progress / file_count
DownloadBase.set_progress(build_application_zip, progress,
100)
# Integrity check that all media files present in media_suite.xml were added to the zip
if include_multimedia_files and include_index_files and toggles.CAUTIOUS_MULTIMEDIA.enabled(
app.domain):
with open(fpath, 'rb') as tmp:
with zipfile.ZipFile(tmp, "r") as z:
media_suites = [
f for f in z.namelist()
if re.search(r'\bmedia_suite.xml\b', f)
]
if len(media_suites) != 1:
message = _(
'Could not identify media_suite.xml in CCZ')
errors.append(message)
notify_exception(None, "[ICDS-291] {}".format(message))
else:
with z.open(media_suites[0]) as media_suite:
from corehq.apps.app_manager.xform import parse_xml
parsed = parse_xml(media_suite.read())
resources = {
node.text
for node in parsed.findall(
"media/resource/location[@authority='local']"
)
}
names = z.namelist()
missing = [
r for r in resources
if re.sub(r'^\.\/', '', r) not in names
]
if missing:
soft_assert(notify_admins=True)(
False, '[ICDS-291] Files missing from CCZ',
[{
'missing file count': len(missing),
'app_id': app._id,
'version': app.version,
'build_profile_id': build_profile_id,
}, {
'files': missing,
}])
errors += [
_('Media file missing from CCZ: {}').format(r)
for r in missing
]
if errors:
os.remove(fpath)
update_task_state(build_application_zip, states.FAILURE,
{'errors': errors})
raise Ignore(
) # We want the task to fail hard, so ignore any future updates to it
else:
DownloadBase.set_progress(build_application_zip,
initial_progress + file_progress, 100)
common_kwargs = {
'mimetype':
'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition':
'attachment; filename="{fname}"'.format(fname=filename),
'download_id':
download_id,
'expiry': (1 * 60 * 60),
}
if use_transfer:
expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs)
else:
expose_cached_download(
FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs)
DownloadBase.set_progress(build_application_zip, 100, 100)