def set_xmlns_on_form(form_id, xmlns, app_build, log_file, dry_run):
"""
Set the xmlns on a form and all the corresponding forms in the saved builds
that are copies of app.
(form is an app_manager.models.Form)
"""
try:
form_in_build = app_build.get_form(form_id)
except FormNotFoundException:
return
if form_in_build.xmlns == "undefined" or form_in_build.source.count('xmlns="undefined"') > 0:
if form_in_build.xmlns != "undefined":
assert form_in_build.xmlns == xmlns
xml = form_in_build.source
wrapped_xml = XForm(xml)
data = wrapped_xml.data_node.render().decode('utf-8')
data = data.replace("undefined", xmlns, 1)
wrapped_xml.instance_node.remove(wrapped_xml.data_node.xml)
wrapped_xml.instance_node.append(parse_xml(data))
new_xml = wrapped_xml.render().decode('utf-8')
form_in_build.source = new_xml
form_in_build.form_migrated_from_undefined_xmlns = datetime.utcnow()
log_file.write(
"New xmlns for form {form_id} in app {app_build._id} is {new_xmlns}\n".format(
form_id=form_id,
app_build=app_build,
new_xmlns=xmlns
))
if not dry_run:
app_build.save() # Magic happens on save
def change_xmlns(xform, replacing):
data = xform.data_node.render()
xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id()
data = data.replace(replacing, xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
xml = xform.render()
return xform, xml
def save_xform(app, form, xml):
try:
xform = XForm(xml)
except XFormError:
pass
else:
duplicates = app.get_xmlns_map()[xform.data_node.tag_xmlns]
for duplicate in duplicates:
if form == duplicate:
continue
else:
data = xform.data_node.render()
xmlns = "http://openrosa.org/formdesigner/%s" % form.get_unique_id()
data = data.replace(xform.data_node.tag_xmlns, xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
xml = xform.render()
break
form.source = xml
def change_xmlns(xform, old_xmlns, new_xmlns):
data = xform.data_node.render().decode('utf-8')
data = data.replace(old_xmlns, new_xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
return xform.render()
def change_xmlns(xform, old_xmlns, new_xmlns):
data = xform.data_node.render().decode('utf-8')
data = data.replace(old_xmlns, new_xmlns, 1)
xform.instance_node.remove(xform.data_node.xml)
xform.instance_node.append(parse_xml(data))
return xform.render()
def build_application_zip(include_multimedia_files, include_index_files, app,
download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip",
download_targeted_version=False):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
initial_progress = 10 # early on indicate something is happening
file_progress = 50.0 # arbitrarily say building files takes half the total time
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format(
app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version,
build_profile_id
))
if download_targeted_version:
fpath += '-targeted'
else:
dummy, fpath = tempfile.mkstemp()
DownloadBase.set_progress(build_application_zip, initial_progress, 100)
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
files, errors, file_count = iter_app_files(
app, include_multimedia_files, include_index_files, build_profile_id,
download_targeted_version=download_targeted_version,
)
if toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain):
manifest = json.dumps({
'include_multimedia_files': include_multimedia_files,
'include_index_files': include_index_files,
'download_id': download_id,
'build_profile_id': build_profile_id,
'compress_zip': compress_zip,
'filename': filename,
'download_targeted_version': download_targeted_version,
'app': app.to_json(),
}, indent=4)
files = itertools.chain(files, [('manifest.json', manifest)])
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
progress = initial_progress
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
progress += file_progress / file_count
DownloadBase.set_progress(build_application_zip, progress, 100)
# Integrity check that all media files present in media_suite.xml were added to the zip
if include_multimedia_files and include_index_files and toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain):
with open(fpath, 'rb') as tmp:
with zipfile.ZipFile(tmp, "r") as z:
media_suites = [f for f in z.namelist() if re.search(r'\bmedia_suite.xml\b', f)]
if len(media_suites) != 1:
message = _('Could not identify media_suite.xml in CCZ')
errors.append(message)
else:
with z.open(media_suites[0]) as media_suite:
from corehq.apps.app_manager.xform import parse_xml
parsed = parse_xml(media_suite.read())
resources = {node.text for node in
parsed.findall("media/resource/location[@authority='local']")}
names = z.namelist()
missing = [r for r in resources if re.sub(r'^\.\/', '', r) not in names]
errors += [_('Media file missing from CCZ: {}').format(r) for r in missing]
if errors:
os.remove(fpath)
raise Exception('\t' + '\t'.join(errors))
else:
DownloadBase.set_progress(build_application_zip, initial_progress + file_progress, 100)
common_kwargs = {
'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename),
'download_id': download_id,
'expiry': (1 * 60 * 60),
}
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs
)
DownloadBase.set_progress(build_application_zip, 100, 100)
def build_application_zip(include_multimedia_files,
include_index_files,
app,
download_id,
build_profile_id=None,
compress_zip=False,
filename="commcare.zip",
download_targeted_version=False):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
initial_progress = 10 # early on indicate something is happening
file_progress = 50.0 # arbitrarily say building files takes half the total time
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(
settings.SHARED_DRIVE_CONF.transfer_dir,
"{}{}{}{}{}".format(app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version, build_profile_id))
if download_targeted_version:
fpath += '-targeted'
else:
dummy, fpath = tempfile.mkstemp()
DownloadBase.set_progress(build_application_zip, initial_progress, 100)
if not (os.path.isfile(fpath)
and use_transfer): # Don't rebuild the file if it is already there
files, errors, file_count = iter_app_files(
app,
include_multimedia_files,
include_index_files,
build_profile_id,
download_targeted_version=download_targeted_version,
)
if toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain):
manifest = json.dumps(
{
'include_multimedia_files': include_multimedia_files,
'include_index_files': include_index_files,
'download_id': download_id,
'build_profile_id': build_profile_id,
'compress_zip': compress_zip,
'filename': filename,
'download_targeted_version': download_targeted_version,
'app': app.to_json(),
},
indent=4)
files = itertools.chain(files, [('manifest.json', manifest)])
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
progress = initial_progress
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
progress += file_progress / file_count
DownloadBase.set_progress(build_application_zip, progress,
100)
# Integrity check that all media files present in media_suite.xml were added to the zip
if include_multimedia_files and include_index_files and toggles.CAUTIOUS_MULTIMEDIA.enabled(
app.domain):
with open(fpath, 'rb') as tmp:
with zipfile.ZipFile(tmp, "r") as z:
media_suites = [
f for f in z.namelist()
if re.search(r'\bmedia_suite.xml\b', f)
]
if len(media_suites) != 1:
message = _(
'Could not identify media_suite.xml in CCZ')
errors.append(message)
notify_exception(None, "[ICDS-291] {}".format(message))
else:
with z.open(media_suites[0]) as media_suite:
from corehq.apps.app_manager.xform import parse_xml
parsed = parse_xml(media_suite.read())
resources = {
node.text
for node in parsed.findall(
"media/resource/location[@authority='local']"
)
}
names = z.namelist()
missing = [
r for r in resources
if re.sub(r'^\.\/', '', r) not in names
]
if missing:
soft_assert(notify_admins=True)(
False, '[ICDS-291] Files missing from CCZ',
[{
'missing file count': len(missing),
'app_id': app._id,
'version': app.version,
'build_profile_id': build_profile_id,
}, {
'files': missing,
}])
errors += [
_('Media file missing from CCZ: {}').format(r)
for r in missing
]
if errors:
os.remove(fpath)
update_task_state(build_application_zip, states.FAILURE,
{'errors': errors})
raise Ignore(
) # We want the task to fail hard, so ignore any future updates to it
else:
DownloadBase.set_progress(build_application_zip,
initial_progress + file_progress, 100)
common_kwargs = {
'mimetype':
'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition':
'attachment; filename="{fname}"'.format(fname=filename),
'download_id':
download_id,
'expiry': (1 * 60 * 60),
}
if use_transfer:
expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs)
else:
expose_cached_download(
FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs)
DownloadBase.set_progress(build_application_zip, 100, 100)