def build_application_zip(include_multimedia_files,
include_index_files,
app,
download_id,
build_profile_id=None,
compress_zip=False,
filename="commcare.zip"):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(
settings.SHARED_DRIVE_CONF.transfer_dir,
"{}{}{}{}{}".format(app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version, build_profile_id))
else:
_, fpath = tempfile.mkstemp()
if not (os.path.isfile(fpath)
and use_transfer): # Don't rebuild the file if it is already there
files, errors = iter_app_files(app, include_multimedia_files,
include_index_files, build_profile_id)
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
common_kwargs = dict(
mimetype='application/zip'
if compress_zip else 'application/x-zip-compressed',
content_disposition='attachment; filename="{fname}"'.format(
fname=filename),
download_id=download_id,
)
if use_transfer:
expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
file_extension=file_extention_from_filename(filename),
**common_kwargs)
DownloadBase.set_progress(build_application_zip, 100, 100)
return {
"errors": errors,
}
def build_application_zip(include_multimedia_files, include_index_files, app,
download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip"):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format(
app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version,
build_profile_id
))
else:
_, fpath = tempfile.mkstemp()
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
files, errors = iter_app_files(app, include_multimedia_files, include_index_files, build_profile_id)
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
common_kwargs = dict(
mimetype='application/zip' if compress_zip else 'application/x-zip-compressed',
content_disposition='attachment; filename="{fname}"'.format(fname=filename),
download_id=download_id,
)
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
file_extension=file_extention_from_filename(filename),
**common_kwargs
)
DownloadBase.set_progress(build_application_zip, 100, 100)
return {
"errors": errors,
}
def test_user_auth_required_access_denied(self):
ref = expose_file_download(self.path, expiry=60, owner_ids=['foo'])
response = self.client.get(
reverse('retrieve_download', args=[ref.download_id]) + "?get_file")
self.assertEqual(response.status_code, 403)
ref = expose_file_download(self.path,
expiry=60,
owner_ids=['foo'],
use_transfer=True)
response = self.client.get(
reverse('retrieve_download', args=[ref.download_id]) + "?get_file")
self.assertEqual(response.status_code, 403)
def _expose_download_link(fpath, filename, compress_zip, download_id):
common_kwargs = {
'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename),
'download_id': download_id,
'expiry': (1 * 60 * 60),
}
if settings.SHARED_DRIVE_CONF.transfer_enabled:
expose_file_download(fpath, use_transfer=True, **common_kwargs)
else:
expose_cached_download(FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs)
def test_user_auth_required_access_allowed(self):
ref = expose_file_download(self.path,
expiry=60,
owner_ids=[self.couch_user.get_id])
response = self.client.get(
reverse('retrieve_download', args=[ref.download_id]) + "?get_file")
self.assertEqual(next(response.streaming_content), b'content')
ref = expose_file_download(self.path,
expiry=60,
owner_ids=[self.couch_user.get_id],
use_transfer=True)
response = self.client.get(
reverse('retrieve_download', args=[ref.download_id]) + "?get_file")
self.assertEqual(next(response.streaming_content), b'content')
def _expose_download(fpath, use_transfer, zip_name, download_id, num_forms):
common_kwargs = dict(
mimetype='application/zip',
content_disposition='attachment; filename="{fname}.zip"'.format(
fname=zip_name),
download_id=download_id,
)
if use_transfer:
expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
file_extension=file_extention_from_filename(fpath),
**common_kwargs)
DownloadBase.set_progress(build_form_multimedia_zip, num_forms, num_forms)
def _expose_download(fpath, use_transfer, zip_name, download_id, num_forms):
common_kwargs = dict(
mimetype='application/zip',
content_disposition='attachment; filename="{fname}.zip"'.format(fname=zip_name),
download_id=download_id,
)
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
file_extension=file_extention_from_filename(fpath),
**common_kwargs
)
DownloadBase.set_progress(build_form_multimedia_zip, num_forms, num_forms)
def build_form_multimedia_zip(domain, xmlns, startdate, enddate, app_id, export_id, zip_name, download_id):
def find_question_id(form, value):
for k, v in form.iteritems():
if isinstance(v, dict):
ret = find_question_id(v, value)
if ret:
return [k] + ret
else:
if v == value:
return [k]
return None
def filename(form_info, question_id, extension):
fname = u"%s-%s-%s-%s%s"
if form_info["cases"]:
fname = u"-".join(form_info["cases"]) + u"-" + fname
return fname % (form_info["name"], unidecode(question_id), form_info["user"], form_info["id"], extension)
case_ids = set()
def extract_form_info(form, properties=None, case_ids=case_ids):
unknown_number = 0
meta = form["form"].get("meta", dict())
# get case ids
case_blocks = extract_case_blocks(form)
cases = {c["@case_id"] for c in case_blocks}
case_ids |= cases
form_info = {
"form": form,
"attachments": list(),
"name": form["form"].get("@name", "unknown form"),
"user": meta.get("username", "unknown_user"),
"cases": cases,
"id": form["_id"],
}
for k, v in form["_attachments"].iteritems():
if v["content_type"] == "text/xml":
continue
try:
question_id = unicode(u"-".join(find_question_id(form["form"], k)))
except TypeError:
question_id = unicode(u"unknown" + unicode(unknown_number))
unknown_number += 1
if not properties or question_id in properties:
extension = unicode(os.path.splitext(k)[1])
form_info["attachments"].append(
{
"size": v["length"],
"name": k,
"question_id": question_id,
"extension": extension,
"timestamp": parse(form["received_on"]).timetuple(),
}
)
return form_info
key = [domain, app_id, xmlns]
form_ids = {
f["id"]
for f in XFormInstance.get_db().view(
"attachments/attachments", start_key=key + [startdate], end_key=key + [enddate, {}], reduce=False
)
}
properties = set()
if export_id:
schema = FormExportSchema.get(export_id)
for table in schema.tables:
# - in question id is replaced by . in excel exports
properties |= {c.display.replace(".", "-") for c in table.columns}
if not app_id:
zip_name = "Unrelated Form"
forms_info = list()
for form in iter_docs(XFormInstance.get_db(), form_ids):
if not zip_name:
zip_name = unidecode(form["form"].get("@name", "unknown form"))
forms_info.append(extract_form_info(form, properties))
num_forms = len(forms_info)
DownloadBase.set_progress(build_form_multimedia_zip, 0, num_forms)
# get case names
case_id_to_name = {c: c for c in case_ids}
for case in iter_docs(CommCareCase.get_db(), case_ids):
if case["name"]:
case_id_to_name[case["_id"]] = case["name"]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
params = "_".join(map(str, [xmlns, startdate, enddate, export_id, num_forms]))
fname = "{}-{}".format(app_id, hashlib.md5(params).hexdigest())
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, fname)
else:
_, fpath = tempfile.mkstemp()
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
with open(fpath, "wb") as zfile:
with zipfile.ZipFile(zfile, "w") as z:
for form_number, form_info in enumerate(forms_info):
f = XFormInstance.wrap(form_info["form"])
form_info["cases"] = {case_id_to_name[case_id] for case_id in form_info["cases"]}
for a in form_info["attachments"]:
fname = filename(form_info, a["question_id"], a["extension"])
zi = zipfile.ZipInfo(fname, a["timestamp"])
z.writestr(zi, f.fetch_attachment(a["name"], stream=True).read(), zipfile.ZIP_STORED)
DownloadBase.set_progress(build_form_multimedia_zip, form_number + 1, num_forms)
common_kwargs = dict(
mimetype="application/zip",
content_disposition='attachment; filename="{fname}.zip"'.format(fname=zip_name),
download_id=download_id,
)
if use_transfer:
expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
file_extension=file_extention_from_filename(fpath),
**common_kwargs
)
DownloadBase.set_progress(build_form_multimedia_zip, num_forms, num_forms)
def test_no_auth_needed(self):
ref = expose_file_download(self.path, expiry=60)
response = self.client.get(
reverse('retrieve_download', args=[ref.download_id]) + "?get_file")
self.assertEqual(next(response.streaming_content), b'content')
def build_application_zip(include_multimedia_files, include_index_files, app,
download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip",
download_targeted_version=False):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
initial_progress = 10 # early on indicate something is happening
file_progress = 50.0 # arbitrarily say building files takes half the total time
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format(
app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version,
build_profile_id
))
if download_targeted_version:
fpath += '-targeted'
else:
_, fpath = tempfile.mkstemp()
DownloadBase.set_progress(build_application_zip, initial_progress, 100)
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
files, errors, file_count = iter_app_files(
app, include_multimedia_files, include_index_files, build_profile_id,
download_targeted_version=download_targeted_version,
)
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
progress = initial_progress
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
progress += file_progress / file_count
DownloadBase.set_progress(build_application_zip, progress, 100)
else:
DownloadBase.set_progress(build_application_zip, initial_progress + file_progress, 100)
common_kwargs = {
'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename),
'download_id': download_id,
'expiry': (1 * 60 * 60),
}
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs
)
DownloadBase.set_progress(build_application_zip, 100, 100)
return {
"errors": errors,
}
def build_form_multimedia_zip(domain, xmlns, startdate, enddate, app_id, export_id, zip_name, download_id):
def find_question_id(form, value):
for k, v in form.iteritems():
if isinstance(v, dict):
ret = find_question_id(v, value)
if ret:
return [k] + ret
else:
if v == value:
return [k]
return None
def filename(form_info, question_id, extension):
fname = u"%s-%s-%s-%s%s"
if form_info['cases']:
fname = u'-'.join(form_info['cases']) + u'-' + fname
return fname % (form_info['name'],
unidecode(question_id),
form_info['user'],
form_info['id'], extension)
case_ids = set()
def extract_form_info(form, properties=None, case_ids=case_ids):
unknown_number = 0
meta = form['form'].get('meta', dict())
# get case ids
case_blocks = extract_case_blocks(form)
cases = {c['@case_id'] for c in case_blocks}
case_ids |= cases
form_info = {
'form': form,
'attachments': list(),
'name': form['form'].get('@name', 'unknown form'),
'user': meta.get('username', 'unknown_user'),
'cases': cases,
'id': form['_id']
}
for k, v in form['_attachments'].iteritems():
if v['content_type'] == 'text/xml':
continue
try:
question_id = unicode(u'-'.join(find_question_id(form['form'], k)))
except TypeError:
question_id = unicode(u'unknown' + unicode(unknown_number))
unknown_number += 1
if not properties or question_id in properties:
extension = unicode(os.path.splitext(k)[1])
form_info['attachments'].append({
'size': v['length'],
'name': k,
'question_id': question_id,
'extension': extension,
'timestamp': parse(form['received_on']).timetuple(),
})
return form_info
key = [domain, app_id, xmlns]
form_ids = {f['id'] for f in XFormInstance.get_db().view("attachments/attachments",
start_key=key + [startdate],
end_key=key + [enddate, {}],
reduce=False)}
properties = set()
if export_id:
schema = FormExportSchema.get(export_id)
for table in schema.tables:
# - in question id is replaced by . in excel exports
properties |= {c.display.replace('.', '-') for c in table.columns}
if not app_id:
zip_name = 'Unrelated Form'
forms_info = list()
for form in iter_docs(XFormInstance.get_db(), form_ids):
if not zip_name:
zip_name = unidecode(form['form'].get('@name', 'unknown form'))
forms_info.append(extract_form_info(form, properties))
num_forms = len(forms_info)
DownloadBase.set_progress(build_form_multimedia_zip, 0, num_forms)
# get case names
case_id_to_name = {c: c for c in case_ids}
for case in iter_docs(CommCareCase.get_db(), case_ids):
if case['name']:
case_id_to_name[case['_id']] = case['name']
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
params = '_'.join(map(str, [xmlns, startdate, enddate, export_id, num_forms]))
fname = '{}-{}'.format(app_id, hashlib.md5(params).hexdigest())
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, fname)
else:
_, fpath = tempfile.mkstemp()
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
with open(fpath, 'wb') as zfile:
with zipfile.ZipFile(zfile, 'w') as z:
for form_number, form_info in enumerate(forms_info):
f = XFormInstance.wrap(form_info['form'])
form_info['cases'] = {case_id_to_name[case_id] for case_id in form_info['cases']}
for a in form_info['attachments']:
fname = filename(form_info, a['question_id'], a['extension'])
zi = zipfile.ZipInfo(fname, a['timestamp'])
z.writestr(zi, f.fetch_attachment(a['name'], stream=True).read(), zipfile.ZIP_STORED)
DownloadBase.set_progress(build_form_multimedia_zip, form_number + 1, num_forms)
common_kwargs = dict(
mimetype='application/zip',
content_disposition='attachment; filename="{fname}.zip"'.format(fname=zip_name),
download_id=download_id,
)
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
file_extension=file_extention_from_filename(fpath),
**common_kwargs
)
DownloadBase.set_progress(build_form_multimedia_zip, num_forms, num_forms)
def build_application_zip(include_multimedia_files,
include_index_files,
app,
download_id,
build_profile_id=None,
compress_zip=False,
filename="commcare.zip",
download_targeted_version=False):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
initial_progress = 10 # early on indicate something is happening
file_progress = 50.0 # arbitrarily say building files takes half the total time
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(
settings.SHARED_DRIVE_CONF.transfer_dir,
"{}{}{}{}{}".format(app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version, build_profile_id))
if download_targeted_version:
fpath += '-targeted'
else:
_, fpath = tempfile.mkstemp()
DownloadBase.set_progress(build_application_zip, initial_progress, 100)
if not (os.path.isfile(fpath)
and use_transfer): # Don't rebuild the file if it is already there
files, errors, file_count = iter_app_files(
app,
include_multimedia_files,
include_index_files,
build_profile_id,
download_targeted_version=download_targeted_version,
)
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
progress = initial_progress
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
progress += file_progress / file_count
DownloadBase.set_progress(build_application_zip, progress,
100)
else:
DownloadBase.set_progress(build_application_zip,
initial_progress + file_progress, 100)
common_kwargs = {
'mimetype':
'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition':
'attachment; filename="{fname}"'.format(fname=filename),
'download_id':
download_id,
'expiry': (1 * 60 * 60),
}
if use_transfer:
expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs)
else:
expose_cached_download(
FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs)
DownloadBase.set_progress(build_application_zip, 100, 100)
return {
"errors": errors,
}
def dump_users_and_groups(domain, download_id):
from corehq.apps.users.views.mobile.custom_data_fields import UserFieldsView
def _load_memoizer(domain):
group_memoizer = GroupMemoizer(domain=domain)
# load groups manually instead of calling group_memoizer.load_all()
# so that we can detect blank groups
blank_groups = set()
for group in Group.by_domain(domain):
if group.name:
group_memoizer.add_group(group)
else:
blank_groups.add(group)
if blank_groups:
raise GroupNameError(blank_groups=blank_groups)
return group_memoizer
writer = Excel2007ExportWriter()
group_memoizer = _load_memoizer(domain)
location_cache = LocationIdToSiteCodeCache(domain)
user_data_model = CustomDataFieldsDefinition.get_or_create(
domain, UserFieldsView.field_type)
user_headers, user_rows = parse_users(group_memoizer, domain,
user_data_model, location_cache)
group_headers, group_rows = parse_groups(group_memoizer.groups)
headers = [
('users', [user_headers]),
('groups', [group_headers]),
]
rows = [
('users', user_rows),
('groups', group_rows),
]
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
file_path = _get_download_file_path(domain, use_transfer)
writer.open(
header_table=headers,
file=file_path,
)
writer.write(rows)
writer.close()
common_kwargs = dict(
mimetype=Format.from_format('xlsx').mimetype,
content_disposition='attachment; filename="{fname}"'.format(
fname='{}_users.xlsx'.format(domain)),
download_id=download_id,
)
if use_transfer:
expose_file_download(file_path,
use_transfer=use_transfer,
**common_kwargs)
else:
expose_cached_download(FileWrapper(open(file_path, 'r')),
expiry=(1 * 60 * 60),
file_extension='xlsx',
**common_kwargs)
def build_application_zip(include_multimedia_files, include_index_files, app,
download_id, build_profile_id=None, compress_zip=False, filename="commcare.zip",
download_targeted_version=False):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
initial_progress = 10 # early on indicate something is happening
file_progress = 50.0 # arbitrarily say building files takes half the total time
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, "{}{}{}{}{}".format(
app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version,
build_profile_id
))
if download_targeted_version:
fpath += '-targeted'
else:
dummy, fpath = tempfile.mkstemp()
DownloadBase.set_progress(build_application_zip, initial_progress, 100)
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
files, errors, file_count = iter_app_files(
app, include_multimedia_files, include_index_files, build_profile_id,
download_targeted_version=download_targeted_version,
)
if toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain):
manifest = json.dumps({
'include_multimedia_files': include_multimedia_files,
'include_index_files': include_index_files,
'download_id': download_id,
'build_profile_id': build_profile_id,
'compress_zip': compress_zip,
'filename': filename,
'download_targeted_version': download_targeted_version,
'app': app.to_json(),
}, indent=4)
files = itertools.chain(files, [('manifest.json', manifest)])
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
progress = initial_progress
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
progress += file_progress / file_count
DownloadBase.set_progress(build_application_zip, progress, 100)
# Integrity check that all media files present in media_suite.xml were added to the zip
if include_multimedia_files and include_index_files and toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain):
with open(fpath, 'rb') as tmp:
with zipfile.ZipFile(tmp, "r") as z:
media_suites = [f for f in z.namelist() if re.search(r'\bmedia_suite.xml\b', f)]
if len(media_suites) != 1:
message = _('Could not identify media_suite.xml in CCZ')
errors.append(message)
else:
with z.open(media_suites[0]) as media_suite:
from corehq.apps.app_manager.xform import parse_xml
parsed = parse_xml(media_suite.read())
resources = {node.text for node in
parsed.findall("media/resource/location[@authority='local']")}
names = z.namelist()
missing = [r for r in resources if re.sub(r'^\.\/', '', r) not in names]
errors += [_('Media file missing from CCZ: {}').format(r) for r in missing]
if errors:
os.remove(fpath)
raise Exception('\t' + '\t'.join(errors))
else:
DownloadBase.set_progress(build_application_zip, initial_progress + file_progress, 100)
common_kwargs = {
'mimetype': 'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition': 'attachment; filename="{fname}"'.format(fname=filename),
'download_id': download_id,
'expiry': (1 * 60 * 60),
}
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs
)
DownloadBase.set_progress(build_application_zip, 100, 100)
def build_application_zip(include_multimedia_files,
include_index_files,
app,
download_id,
build_profile_id=None,
compress_zip=False,
filename="commcare.zip",
download_targeted_version=False):
from corehq.apps.hqmedia.views import iter_app_files
DownloadBase.set_progress(build_application_zip, 0, 100)
initial_progress = 10 # early on indicate something is happening
file_progress = 50.0 # arbitrarily say building files takes half the total time
errors = []
compression = zipfile.ZIP_DEFLATED if compress_zip else zipfile.ZIP_STORED
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
fpath = os.path.join(
settings.SHARED_DRIVE_CONF.transfer_dir,
"{}{}{}{}{}".format(app._id,
'mm' if include_multimedia_files else '',
'ccz' if include_index_files else '',
app.version, build_profile_id))
if download_targeted_version:
fpath += '-targeted'
else:
dummy, fpath = tempfile.mkstemp()
DownloadBase.set_progress(build_application_zip, initial_progress, 100)
if not (os.path.isfile(fpath)
and use_transfer): # Don't rebuild the file if it is already there
files, errors, file_count = iter_app_files(
app,
include_multimedia_files,
include_index_files,
build_profile_id,
download_targeted_version=download_targeted_version,
)
if toggles.CAUTIOUS_MULTIMEDIA.enabled(app.domain):
manifest = json.dumps(
{
'include_multimedia_files': include_multimedia_files,
'include_index_files': include_index_files,
'download_id': download_id,
'build_profile_id': build_profile_id,
'compress_zip': compress_zip,
'filename': filename,
'download_targeted_version': download_targeted_version,
'app': app.to_json(),
},
indent=4)
files = itertools.chain(files, [('manifest.json', manifest)])
with open(fpath, 'wb') as tmp:
with zipfile.ZipFile(tmp, "w") as z:
progress = initial_progress
for path, data in files:
# don't compress multimedia files
extension = os.path.splitext(path)[1]
file_compression = zipfile.ZIP_STORED if extension in MULTIMEDIA_EXTENSIONS else compression
z.writestr(path, data, file_compression)
progress += file_progress / file_count
DownloadBase.set_progress(build_application_zip, progress,
100)
# Integrity check that all media files present in media_suite.xml were added to the zip
if include_multimedia_files and include_index_files and toggles.CAUTIOUS_MULTIMEDIA.enabled(
app.domain):
with open(fpath, 'rb') as tmp:
with zipfile.ZipFile(tmp, "r") as z:
media_suites = [
f for f in z.namelist()
if re.search(r'\bmedia_suite.xml\b', f)
]
if len(media_suites) != 1:
message = _(
'Could not identify media_suite.xml in CCZ')
errors.append(message)
notify_exception(None, "[ICDS-291] {}".format(message))
else:
with z.open(media_suites[0]) as media_suite:
from corehq.apps.app_manager.xform import parse_xml
parsed = parse_xml(media_suite.read())
resources = {
node.text
for node in parsed.findall(
"media/resource/location[@authority='local']"
)
}
names = z.namelist()
missing = [
r for r in resources
if re.sub(r'^\.\/', '', r) not in names
]
if missing:
soft_assert(notify_admins=True)(
False, '[ICDS-291] Files missing from CCZ',
[{
'missing file count': len(missing),
'app_id': app._id,
'version': app.version,
'build_profile_id': build_profile_id,
}, {
'files': missing,
}])
errors += [
_('Media file missing from CCZ: {}').format(r)
for r in missing
]
if errors:
os.remove(fpath)
update_task_state(build_application_zip, states.FAILURE,
{'errors': errors})
raise Ignore(
) # We want the task to fail hard, so ignore any future updates to it
else:
DownloadBase.set_progress(build_application_zip,
initial_progress + file_progress, 100)
common_kwargs = {
'mimetype':
'application/zip' if compress_zip else 'application/x-zip-compressed',
'content_disposition':
'attachment; filename="{fname}"'.format(fname=filename),
'download_id':
download_id,
'expiry': (1 * 60 * 60),
}
if use_transfer:
expose_file_download(fpath, use_transfer=use_transfer, **common_kwargs)
else:
expose_cached_download(
FileWrapper(open(fpath, 'rb')),
file_extension=file_extention_from_filename(filename),
**common_kwargs)
DownloadBase.set_progress(build_application_zip, 100, 100)
def build_form_multimedia_zip(domain, xmlns, startdate, enddate, app_id, export_id, zip_name, download_id):
def find_question_id(form, value):
for k, v in form.iteritems():
if isinstance(v, dict):
ret = find_question_id(v, value)
if ret:
return [k] + ret
else:
if v == value:
return [k]
return None
def filename(form_info, question_id, extension):
fname = u"%s-%s-%s-%s%s"
if form_info['cases']:
fname = u'-'.join(form_info['cases']) + u'-' + fname
return fname % (form_info['name'],
unidecode(question_id),
form_info['user'],
form_info['id'], extension)
case_ids = set()
def extract_form_info(form, properties=None, case_ids=case_ids):
unknown_number = 0
meta = form['form'].get('meta', dict())
# get case ids
case_blocks = extract_case_blocks(form)
cases = {c['@case_id'] for c in case_blocks}
case_ids |= cases
form_info = {
'form': form,
'attachments': list(),
'name': form['form'].get('@name', 'unknown form'),
'user': meta.get('username', 'unknown_user'),
'cases': cases,
'id': form['_id']
}
for k, v in form['_attachments'].iteritems():
if v['content_type'] == 'text/xml':
continue
try:
question_id = unicode(u'-'.join(find_question_id(form['form'], k)))
except TypeError:
question_id = unicode(u'unknown' + unicode(unknown_number))
unknown_number += 1
if not properties or question_id in properties:
extension = unicode(os.path.splitext(k)[1])
form_info['attachments'].append({
'size': v['length'],
'name': k,
'question_id': question_id,
'extension': extension,
'timestamp': parse(form['received_on']).timetuple(),
})
return form_info
key = [domain, app_id, xmlns]
form_ids = {f['id'] for f in XFormInstance.get_db().view("attachments/attachments",
start_key=key + [startdate],
end_key=key + [enddate, {}],
reduce=False)}
properties = set()
if export_id:
schema = FormExportSchema.get(export_id)
for table in schema.tables:
# - in question id is replaced by . in excel exports
properties |= {c.display.replace('.', '-') for c in table.columns}
if not app_id:
zip_name = 'Unrelated Form'
forms_info = list()
for form in iter_docs(XFormInstance.get_db(), form_ids):
if not zip_name:
zip_name = unidecode(form['form'].get('@name', 'unknown form'))
forms_info.append(extract_form_info(form, properties))
num_forms = len(forms_info)
DownloadBase.set_progress(build_form_multimedia_zip, 0, num_forms)
# get case names
case_id_to_name = {c: c for c in case_ids}
for case in iter_docs(CommCareCase.get_db(), case_ids):
if case['name']:
case_id_to_name[case['_id']] = case['name']
use_transfer = settings.SHARED_DRIVE_CONF.transfer_enabled
if use_transfer:
params = '_'.join(map(str, [xmlns, startdate, enddate, export_id, num_forms]))
fname = '{}-{}'.format(app_id, hashlib.md5(params).hexdigest())
fpath = os.path.join(settings.SHARED_DRIVE_CONF.transfer_dir, fname)
else:
_, fpath = tempfile.mkstemp()
if not (os.path.isfile(fpath) and use_transfer): # Don't rebuild the file if it is already there
with open(fpath, 'wb') as zfile:
with zipfile.ZipFile(zfile, 'w') as z:
for form_number, form_info in enumerate(forms_info):
f = XFormInstance.wrap(form_info['form'])
form_info['cases'] = {case_id_to_name[case_id] for case_id in form_info['cases']}
for a in form_info['attachments']:
fname = filename(form_info, a['question_id'], a['extension'])
zi = zipfile.ZipInfo(fname, a['timestamp'])
z.writestr(zi, f.fetch_attachment(a['name'], stream=True).read(), zipfile.ZIP_STORED)
DownloadBase.set_progress(build_form_multimedia_zip, form_number + 1, num_forms)
common_kwargs = dict(
mimetype='application/zip',
content_disposition='attachment; filename="{fname}.zip"'.format(fname=zip_name),
download_id=download_id,
)
if use_transfer:
expose_file_download(
fpath,
use_transfer=use_transfer,
**common_kwargs
)
else:
expose_cached_download(
FileWrapper(open(fpath)),
expiry=(1 * 60 * 60),
**common_kwargs
)
DownloadBase.set_progress(build_form_multimedia_zip, num_forms, num_forms)