def _build_ccz_files(build, build_profile_id, include_multimedia_files, include_index_files, download_id, compress_zip, filename, download_targeted_version): from corehq.apps.hqmedia.views import iter_app_files files, errors, file_count = iter_app_files( build, include_multimedia_files, include_index_files, build_profile_id, download_targeted_version=download_targeted_version, ) if toggles.CAUTIOUS_MULTIMEDIA.enabled(build.domain): manifest = json.dumps( { 'include_multimedia_files': include_multimedia_files, 'include_index_files': include_index_files, 'download_id': download_id, 'build_profile_id': build_profile_id, 'compress_zip': compress_zip, 'filename': filename, 'download_targeted_version': download_targeted_version, 'app': build.to_json(), }, indent=4) files = itertools.chain(files, [('manifest.json', manifest)]) return files, errors, file_count
def build_application_zip(include_multimedia_files, include_index_files, app, download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip"): from corehq.apps.hqmedia.views import iter_app_files DownloadBase.set_progress(build_application_zip, 0, 100) errors = [] compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled if use_transfer: fpath = os.path.join( settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format(app._id, 'mm' if include_multimedia_files else '', 'ccz' if include_index_files else '', app.version, build_profile_id)) else: _, fpath = tempfile.mkstemp() if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there files, errors = iter_app_files(app, include_multimedia_files, include_index_files, build_profile_id) with open(fpath, 'wb') as tmp: with zipfile.ZipFile(tmp, "w") as z: for path, data in files: # don't compress multimedia files extension = os.path.splitext(path)[1] file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression z.writestr(path, data, file_compression) common_kwargs = dict( mimetype='application/zip' if compress_zip else 'application/x-zip-compressed', content_disposition='attachment; filename="{fname}"'.format( fname=filename), download_id=download_id, ) if use_transfer: expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs) else: expose_cached_download( FileWrapper(open(fpath)), expiry=(1 * 60 * 60), file_extension=file_extention_from_filename(filename), **common_kwargs) DownloadBase.set_progress(build_application_zip, 100, 100) return { "errors": errors, }
def build_application_zip(include_multimedia_files, include_index_files, app, download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip"): from corehq.apps.hqmedia.views import iter_app_files DownloadBase.set_progress(build_application_zip, 0, 100) errors = [] compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled if use_transfer: fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format( app._id, 'mm' if include_multimedia_files else '', 'ccz' if include_index_files else '', app.version, build_profile_id )) else: _, fpath = tempfile.mkstemp() if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there files, errors = iter_app_files(app, include_multimedia_files, include_index_files, build_profile_id) with open(fpath, 'wb') as tmp: with zipfile.ZipFile(tmp, "w") as z: for path, data in files: # don't compress multimedia files extension = os.path.splitext(path)[1] file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression z.writestr(path, data, file_compression) common_kwargs = dict( mimetype='application/zip' if compress_zip else 'application/x-zip-compressed', content_disposition='attachment; filename="{fname}"'.format(fname=filename), download_id=download_id, ) if use_transfer: expose_file_download( fpath, use_transfer=use_transfer, **common_kwargs ) else: expose_cached_download( FileWrapper(open(fpath)), expiry=(1 * 60 * 60), file_extension=file_extention_from_filename(filename), **common_kwargs ) DownloadBase.set_progress(build_application_zip, 100, 100) return { "errors": errors, }
def build_application_zip(include_multimedia_files, include_index_files, app, download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip", download_targeted_version=False): from corehq.apps.hqmedia.views import iter_app_files DownloadBase.set_progress(build_application_zip, 0, 100) initial_progress = 10 # early on indicate something is happening file_progress = 50.0 # arbitrarily say building files takes half the total time errors = [] compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled if use_transfer: fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format( app._id, 'mm' if include_multimedia_files else '', 'ccz' if include_index_files else '', app.version, build_profile_id )) if download_targeted_version: fpath += '-targeted' else: _, fpath = tempfile.mkstemp() DownloadBase.set_progress(build_application_zip, initial_progress, 100) if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there files, errors, file_count = iter_app_files( app, include_multimedia_files, include_index_files, build_profile_id, download_targeted_version=download_targeted_version, ) with open(fpath, 'wb') as tmp: with zipfile.ZipFile(tmp, "w") as z: progress = initial_progress for path, data in files: # don't compress multimedia files extension = os.path.splitext(path)[1] file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression z.writestr(path, data, file_compression) progress += file_progress / file_count DownloadBase.set_progress(build_application_zip, progress, 100) else: DownloadBase.set_progress(build_application_zip, initial_progress + file_progress, 100) common_kwargs = { 'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed', 'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename), 'download_id': download_id, 'expiry': (1 * 60 * 60), } if use_transfer: expose_file_download( fpath, use_transfer=use_transfer, **common_kwargs ) else: expose_cached_download( FileWrapper(open(fpath, 'rb')), file_extension=file_extention_from_filename(filename), **common_kwargs ) DownloadBase.set_progress(build_application_zip, 100, 100) return { "errors": errors, }
def build_application_zip(include_multimedia_files, include_index_files, app, download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip", download_targeted_version=False): from corehq.apps.hqmedia.views import iter_app_files DownloadBase.set_progress(build_application_zip, 0, 100) initial_progress = 10 # early on indicate something is happening file_progress = 50.0 # arbitrarily say building files takes half the total time errors = [] compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled if use_transfer: fpath = os.path.join( settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format(app._id, 'mm' if include_multimedia_files else '', 'ccz' if include_index_files else '', app.version, build_profile_id)) if download_targeted_version: fpath += '-targeted' else: _, fpath = tempfile.mkstemp() DownloadBase.set_progress(build_application_zip, initial_progress, 100) if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there files, errors, file_count = iter_app_files( app, include_multimedia_files, include_index_files, build_profile_id, download_targeted_version=download_targeted_version, ) with open(fpath, 'wb') as tmp: with zipfile.ZipFile(tmp, "w") as z: progress = initial_progress for path, data in files: # don't compress multimedia files extension = os.path.splitext(path)[1] file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression z.writestr(path, data, file_compression) progress += file_progress / file_count DownloadBase.set_progress(build_application_zip, progress, 100) else: DownloadBase.set_progress(build_application_zip, initial_progress + file_progress, 100) common_kwargs = { 'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed', 'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename), 'download_id': download_id, 'expiry': (1 * 60 * 60), } if use_transfer: expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs) else: expose_cached_download( FileWrapper(open(fpath, 'rb')), file_extension=file_extention_from_filename(filename), **common_kwargs) DownloadBase.set_progress(build_application_zip, 100, 100) return { "errors": errors, }
def build_application_zip(include_multimedia_files, include_index_files, app, download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip", download_targeted_version=False): from corehq.apps.hqmedia.views import iter_app_files DownloadBase.set_progress(build_application_zip, 0, 100) initial_progress = 10 # early on indicate something is happening file_progress = 50.0 # arbitrarily say building files takes half the total time errors = [] compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled if use_transfer: fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format( app._id, 'mm' if include_multimedia_files else '', 'ccz' if include_index_files else '', app.version, build_profile_id )) if download_targeted_version: fpath += '-targeted' else: dummy, fpath = tempfile.mkstemp() DownloadBase.set_progress(build_application_zip, initial_progress, 100) if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there files, errors, file_count = iter_app_files( app, include_multimedia_files, include_index_files, build_profile_id, download_targeted_version=download_targeted_version, ) if toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain): manifest = json.dumps({ 'include_multimedia_files': include_multimedia_files, 'include_index_files': include_index_files, 'download_id': download_id, 'build_profile_id': build_profile_id, 'compress_zip': compress_zip, 'filename': filename, 'download_targeted_version': download_targeted_version, 'app': app.to_json(), }, indent=4) files = itertools.chain(files, [('manifest.json', manifest)]) with open(fpath, 'wb') as tmp: with zipfile.ZipFile(tmp, "w") as z: progress = initial_progress for path, data in files: # don't compress multimedia files extension = os.path.splitext(path)[1] file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression z.writestr(path, data, file_compression) progress += file_progress / file_count DownloadBase.set_progress(build_application_zip, progress, 100) # Integrity check that all media files present in media_suite.xml were added to the zip if include_multimedia_files and include_index_files and toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain): with open(fpath, 'rb') as tmp: with zipfile.ZipFile(tmp, "r") as z: media_suites = [f for f in z.namelist() if re.search(r'\bmedia_suite.xml\b', f)] if len(media_suites) != 1: message = _('Could not identify media_suite.xml in CCZ') errors.append(message) else: with z.open(media_suites[0]) as media_suite: from corehq.apps.app_manager.xform import parse_xml parsed = parse_xml(media_suite.read()) resources = {node.text for node in parsed.findall("media/resource/location[@authority='local']")} names = z.namelist() missing = [r for r in resources if re.sub(r'^\.\/', '', r) not in names] errors += [_('Media file missing from CCZ: {}').format(r) for r in missing] if errors: os.remove(fpath) raise Exception('\t' + '\t'.join(errors)) else: DownloadBase.set_progress(build_application_zip, initial_progress + file_progress, 100) common_kwargs = { 'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed', 'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename), 'download_id': download_id, 'expiry': (1 * 60 * 60), } if use_transfer: expose_file_download( fpath, use_transfer=use_transfer, **common_kwargs ) else: expose_cached_download( FileWrapper(open(fpath, 'rb')), file_extension=file_extention_from_filename(filename), **common_kwargs ) DownloadBase.set_progress(build_application_zip, 100, 100)
def build_application_zip(include_multimedia_files, include_index_files, app, download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip", download_targeted_version=False): from corehq.apps.hqmedia.views import iter_app_files DownloadBase.set_progress(build_application_zip, 0, 100) initial_progress = 10 # early on indicate something is happening file_progress = 50.0 # arbitrarily say building files takes half the total time errors = [] compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled if use_transfer: fpath = os.path.join( settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format(app._id, 'mm' if include_multimedia_files else '', 'ccz' if include_index_files else '', app.version, build_profile_id)) if download_targeted_version: fpath += '-targeted' else: dummy, fpath = tempfile.mkstemp() DownloadBase.set_progress(build_application_zip, initial_progress, 100) if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there files, errors, file_count = iter_app_files( app, include_multimedia_files, include_index_files, build_profile_id, download_targeted_version=download_targeted_version, ) if toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain): manifest = json.dumps( { 'include_multimedia_files': include_multimedia_files, 'include_index_files': include_index_files, 'download_id': download_id, 'build_profile_id': build_profile_id, 'compress_zip': compress_zip, 'filename': filename, 'download_targeted_version': download_targeted_version, 'app': app.to_json(), }, indent=4) files = itertools.chain(files, [('manifest.json', manifest)]) with open(fpath, 'wb') as tmp: with zipfile.ZipFile(tmp, "w") as z: progress = initial_progress for path, data in files: # don't compress multimedia files extension = os.path.splitext(path)[1] file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression z.writestr(path, data, file_compression) progress += file_progress / file_count DownloadBase.set_progress(build_application_zip, progress, 100) # Integrity check that all media files present in media_suite.xml were added to the zip if include_multimedia_files and include_index_files and toggles.CAUTIOUS_MULTIMEDIA.enabled( app.domain): with open(fpath, 'rb') as tmp: with zipfile.ZipFile(tmp, "r") as z: media_suites = [ f for f in z.namelist() if re.search(r'\bmedia_suite.xml\b', f) ] if len(media_suites) != 1: message = _( 'Could not identify media_suite.xml in CCZ') errors.append(message) notify_exception(None, "[ICDS-291] {}".format(message)) else: with z.open(media_suites[0]) as media_suite: from corehq.apps.app_manager.xform import parse_xml parsed = parse_xml(media_suite.read()) resources = { node.text for node in parsed.findall( "media/resource/location[@authority='local']" ) } names = z.namelist() missing = [ r for r in resources if re.sub(r'^\.\/', '', r) not in names ] if missing: soft_assert(notify_admins=True)( False, '[ICDS-291] Files missing from CCZ', [{ 'missing file count': len(missing), 'app_id': app._id, 'version': app.version, 'build_profile_id': build_profile_id, }, { 'files': missing, }]) errors += [ _('Media file missing from CCZ: {}').format(r) for r in missing ] if errors: os.remove(fpath) update_task_state(build_application_zip, states.FAILURE, {'errors': errors}) raise Ignore( ) # We want the task to fail hard, so ignore any future updates to it else: DownloadBase.set_progress(build_application_zip, initial_progress + file_progress, 100) common_kwargs = { 'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed', 'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename), 'download_id': download_id, 'expiry': (1 * 60 * 60), } if use_transfer: expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs) else: expose_cached_download( FileWrapper(open(fpath, 'rb')), file_extension=file_extention_from_filename(filename), **common_kwargs) DownloadBase.set_progress(build_application_zip, 100, 100)