def test_multimedia(self):
from corehq.apps.hqmedia.models import CommCareAudio, CommCareImage, CommCareVideo
image_path = os.path.join('corehq', 'apps', 'hqwebapp', 'static', 'hqwebapp', 'images', 'commcare-hq-logo.png')
with open(image_path, 'r') as f:
image_data = f.read()
image = CommCareImage.get_by_data(image_data)
image.attach_data(image_data, original_filename='logo.png')
image.add_domain(self.domain_name)
self.assertEqual(image_data, image.get_display_file(False))
audio_data = 'fake audio data'
audio = CommCareAudio.get_by_data(audio_data)
audio.attach_data(audio_data, original_filename='tr-la-la.mp3')
audio.add_domain(self.domain_name)
self.assertEqual(audio_data, audio.get_display_file(False))
video_data = 'fake video data'
video = CommCareVideo.get_by_data(video_data)
video.attach_data(video_data, 'kittens.mp4')
video.add_domain(self.domain_name)
self.assertEqual(video_data, video.get_display_file(False))
fakedb = self._dump_and_load([image, audio, video])
copied_image = CommCareImage.wrap(fakedb.get(image._id))
copied_audio = CommCareAudio.wrap(fakedb.get(audio._id))
copied_video = CommCareVideo.wrap(fakedb.get(video._id))
self.assertEqual(image_data, copied_image.get_display_file(False))
self.assertEqual(audio_data, copied_audio.get_display_file(False))
self.assertEqual(video_data, copied_video.get_display_file(False))
def test_multimedia(self):
from corehq.apps.hqmedia.models import CommCareAudio, CommCareImage, CommCareVideo
image_path = os.path.join('corehq', 'apps', 'hqwebapp', 'static', 'hqwebapp', 'img', 'commcare-hq-logo.png')
with open(image_path, 'r') as f:
image_data = f.read()
image = CommCareImage.get_by_data(image_data)
image.attach_data(image_data, original_filename='logo.png')
image.add_domain(self.domain_name)
self.assertEqual(image_data, image.get_display_file(False))
audio_data = 'fake audio data'
audio = CommCareAudio.get_by_data(audio_data)
audio.attach_data(audio_data, original_filename='tr-la-la.mp3')
audio.add_domain(self.domain_name)
self.assertEqual(audio_data, audio.get_display_file(False))
video_data = 'fake video data'
video = CommCareVideo.get_by_data(video_data)
video.attach_data(video_data, 'kittens.mp4')
video.add_domain(self.domain_name)
self.assertEqual(video_data, video.get_display_file(False))
fakedb = self._dump_and_load([image, audio, video])
copied_image = CommCareImage.wrap(fakedb.get(image._id))
copied_audio = CommCareAudio.wrap(fakedb.get(audio._id))
copied_video = CommCareVideo.wrap(fakedb.get(video._id))
self.assertEqual(image_data, copied_image.get_display_file(False))
self.assertEqual(audio_data, copied_audio.get_display_file(False))
self.assertEqual(video_data, copied_video.get_display_file(False))
def test_form_media_with_app_profile(self, *args):
# Test that media for languages not in the profile are removed from the media suite
app = Application.wrap(self.get_json('app'))
app.build_profiles['profid'] = BuildProfile(langs=['en'],
name='en-profile')
app.langs = ['en', 'hin']
image_path = 'jr://file/commcare/module0_en.png'
audio_path = 'jr://file/commcare/module0_{}.mp3'
app.get_module(0).set_icon('en', image_path)
# app.get_module(0).case_list_form.set_icon('en', image_path)
app.get_module(0).set_audio('en', audio_path.format('en'))
app.get_module(0).set_audio('hin', audio_path.format('hin'))
app.create_mapping(CommCareImage(_id='123'), image_path, save=False)
app.create_mapping(CommCareAudio(_id='456'),
audio_path.format('en'),
save=False)
app.create_mapping(CommCareAudio(_id='789'),
audio_path.format('hin'),
save=False)
form_xml = self.get_xml('form_with_media_refs').decode('utf-8')
form = app.get_module(0).new_form('form_with_media',
'en',
attachment=form_xml)
xform = form.wrapped_xform()
for i, path in enumerate(xform.media_references(form="audio")):
app.create_mapping(CommCareAudio(_id='form_audio_{}'.format(i)),
path,
save=False)
for i, path in enumerate(xform.media_references(form="image")):
app.create_mapping(CommCareImage(_id='form_image_{}'.format(i)),
path,
save=False)
app.set_media_versions()
app.remove_unused_mappings()
# includes all media
self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite'),
app.create_media_suite())
# includes all app media and only form media for 'en'
self._assertMediaSuiteResourcesEqual(
self.get_xml('form_media_suite_en'),
app.create_media_suite('profid'))
def test_unused_media_removed(self, mock):
image_path = 'jr://file/commcare/image{}_{}.jpg'
audio_path = 'jr://file/commcare/audio{}_{}.mp3'
app = Application.wrap(self.get_json('app'))
app.domain = self.domain
app.save()
for lang in ['en', 'hin']:
for num in ['1', '2', '3', '4']:
app.create_mapping(CommCareImage(_id=num),
image_path.format(num, lang),
save=False)
app.create_mapping(CommCareAudio(_id=num),
audio_path.format(num, lang),
save=False)
app.get_module(0).case_list.show = True
app.get_module(0).case_list.set_icon('en',
image_path.format('4', 'en'))
app.get_module(0).case_list.set_audio('en',
audio_path.format('4', 'en'))
app.get_module(0).set_icon('en', image_path.format('1', 'en'))
app.get_module(0).set_audio('en', audio_path.format('1', 'en'))
app.get_module(0).case_list_form.form_id = app.get_module(0).get_form(
0).unique_id
app.get_module(0).case_list_form.set_icon('en',
image_path.format('2', 'en'))
app.get_module(0).case_list_form.set_audio(
'en', audio_path.format('2', 'en'))
app.get_module(0).get_form(0).set_icon('en',
image_path.format('3', 'en'))
app.get_module(0).get_form(0).set_audio('en',
audio_path.format('3', 'en'))
app.save()
should_contain_media = [image_path.format(num, 'en') for num in [1, 2, 3, 4]] + \
[audio_path.format(num, 'en') for num in [1, 2, 3, 4]]
media_for_removal = [image_path.format(num, 'hin') for num in [1, 2, 3, 4]] + \
[audio_path.format(num, 'hin') for num in [1, 2, 3, 4]]
self.assertTrue(app.get_module(0).uses_media())
self.assertEqual(app.all_media_paths(), set(should_contain_media))
self.assertEqual(set(app.multimedia_map.keys()),
set(should_contain_media + media_for_removal))
app.remove_unused_mappings()
self.assertEqual(set(app.multimedia_map.keys()),
set(should_contain_media))
# test multimedia removed
app.get_module(0).case_list.set_icon('en', '')
app.get_module(0).case_list.set_audio('en', '')
app.get_module(0).set_icon('en', '')
app.get_module(0).set_audio('en', '')
app.get_module(0).case_list_form.set_icon('en', '')
app.get_module(0).case_list_form.set_audio('en', '')
app.get_module(0).get_form(0).set_icon('en', '')
app.get_module(0).get_form(0).set_audio('en', '')
self.assertFalse(list(app.multimedia_map.keys()))
def match_zipped(self, zip, replace_existing_media=True, **kwargs):
unknown_files = []
matched_audio = []
matched_images = []
errors = []
try:
for index, path in enumerate(zip.namelist()):
path = path.strip()
filename = path.lower()
basename = os.path.basename(path.lower())
is_image = False
if not basename:
continue
if self.match_filename:
filename = basename
media = None
form_path = None
data = None
if filename in self.images:
form_path = self.image_paths[self.images.index(filename)]
data = zip.read(path)
media = CommCareImage.get_by_data(data)
is_image = True
elif filename in self.audio:
form_path = self.audio_paths[self.audio.index(filename)]
data = zip.read(path)
media = CommCareAudio.get_by_data(data)
else:
unknown_files.append(path)
if media:
try:
media.attach_data(data,
upload_path=path,
username=self.username,
replace_attachment=replace_existing_media)
media.add_domain(self.domain, owner=True)
media.update_or_add_license(self.domain,
type=kwargs.get('license', ''),
author=kwargs.get('author', ''),
attribution_notes=kwargs.get('attribution_notes', ''))
self.app.create_mapping(media, form_path)
match_map = HQMediaMapItem.format_match_map(form_path, media.doc_type, media._id, path)
if is_image:
matched_images.append(match_map)
else:
matched_audio.append(match_map)
except Exception as e:
errors.append("%s (%s)" % (e, filename))
zip.close()
except Exception as e:
logging.error(e)
errors.append(e.message)
return matched_images, matched_audio, unknown_files, errors
def test_form_media_with_app_profile(self, *args):
# Test that media for languages not in the profile are removed from the media suite
app = Application.wrap(self.get_json('app'))
app.build_profiles = OrderedDict({
'en': BuildProfile(langs=['en'], name='en-profile'),
'hin': BuildProfile(langs=['hin'], name='hin-profile'),
'all': BuildProfile(langs=['en', 'hin'], name='all-profile'),
})
app.langs = ['en', 'hin']
image_path = 'jr://file/commcare/module0_en.png'
audio_path = 'jr://file/commcare/module0_{}.mp3'
app.get_module(0).set_icon('en', image_path)
app.get_module(0).set_audio('en', audio_path.format('en'))
app.get_module(0).set_audio('hin', audio_path.format('hin'))
app.create_mapping(CommCareImage(_id='123'), image_path, save=False)
app.create_mapping(CommCareAudio(_id='456'), audio_path.format('en'), save=False)
app.create_mapping(CommCareAudio(_id='789'), audio_path.format('hin'), save=False)
form_xml = self.get_xml('form_with_media_refs').decode('utf-8')
form = app.get_module(0).new_form('form_with_media', 'en', attachment=form_xml)
xform = form.wrapped_xform()
for i, path in enumerate(reversed(sorted(xform.media_references(form="audio")))):
app.create_mapping(CommCareAudio(_id='form_audio_{}'.format(i)), path, save=False)
for i, path in enumerate(sorted(xform.media_references(form="image"))):
app.create_mapping(CommCareImage(_id='form_image_{}'.format(i)), path, save=False)
app.set_media_versions()
app.remove_unused_mappings()
# includes all media
self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite'), app.create_media_suite())
# generate all suites at once to mimic create_build_files_for_all_app_profiles
suites = {id: app.create_media_suite(build_profile_id=id) for id in app.build_profiles.keys()}
# include all app media and only language-specific form media
self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite_en'), suites['en'])
self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite_hin'), suites['hin'])
self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite_all'), suites['all'])
def save(self, domain, app, username, cache_handler=None):
orig_images, orig_audio, _ = utils.get_multimedia_filenames(app)
form_images = [i.replace(utils.MULTIMEDIA_PREFIX, '').lower().strip() for i in orig_images]
form_audio = [a.replace(utils.MULTIMEDIA_PREFIX, '').lower().strip() for a in orig_audio]
if self.cleaned_data["repopulate_multimedia_map"]:
app.multimedia_map = {}
app.save()
replace_attachment = self.cleaned_data["replace_existing_media"]
zip = self.cleaned_data["zip_file"]
num_files = len(zip.namelist())
if cache_handler:
cache_handler.sync()
cache_handler.data["processed_length"] = num_files
cache_handler.save()
unknown_files = []
for index, path in enumerate(zip.namelist()):
path = path.strip()
path_lower = path.lower()
if path_lower.endswith("/") or path_lower.endswith("\\") or \
path_lower.endswith(".ds_store"):
continue
media = None
form_path = None
data = None
if path_lower in form_images:
form_path = orig_images[form_images.index(path_lower)]
data = zip.read(path)
media = CommCareImage.get_by_data(data)
elif path_lower in form_audio:
form_path = orig_audio[form_audio.index(path_lower)]
data = zip.read(path)
media = CommCareAudio.get_by_data(data)
else:
unknown_files.append(path)
if media:
media.attach_data(data,
upload_path=path,
username=username,
replace_attachment=replace_attachment)
media.add_domain(domain, owner=True)
app.create_mapping(media, form_path)
if cache_handler:
cache_handler.sync()
cache_handler.data["processed"] = index+1
cache_handler.save()
zip.close()
return {"successful": unknown_files}
def search_for_media(request, domain, app_id):
media_type = request.GET['t']
if media_type == 'Image':
files = CommCareImage.search(request.GET['q'])
elif media_type == 'Audio':
files = CommCareAudio.search(request.GET['q'])
else:
raise Http404()
return HttpResponse(json.dumps([
{'url': i.url(),
'licenses': [license.display_name for license in i.licenses],
'tags': [tag for tags in i.tags.values() for tag in tags],
'm_id': i._id} for i in files]))
def search_for_media(request, domain, app_id):
media_type = request.GET['t']
if media_type == 'Image':
files = CommCareImage.search(request.GET['q'])
elif media_type == 'Audio':
files = CommCareAudio.search(request.GET['q'])
else:
raise Http404()
return HttpResponse(json.dumps([
{'url': i.url(),
'licenses': [license.display_name for license in i.licenses],
'tags': [tag for tags in i.tags.values() for tag in tags],
'm_id': i._id} for i in files]))
def test_case_list_media(self):
app = Application.wrap(self.get_json('app'))
app.get_module(0).case_list_form.form_id = app.get_module(0).get_form(0).unique_id
image_path = 'jr://file/commcare/case_list_image.jpg'
audo_path = 'jr://file/commcare/case_list_audo.mp3'
app.get_module(0).case_list_form.set_icon('en', image_path)
app.get_module(0).case_list_form.set_audio('en', audo_path)
app.create_mapping(CommCareImage(_id='123'), image_path, save=False)
app.create_mapping(CommCareAudio(_id='456'), audo_path, save=False)
app.set_media_versions()
self._assertMediaSuiteResourcesEqual(self.get_xml('case_list_media_suite'), app.create_media_suite())
def test_case_list_media(self):
app = Application.wrap(self.get_json('app'))
app.get_module(0).case_list_form.form_id = app.get_module(0).get_form(0).unique_id
image_path = 'jr://file/commcare/case_list_image.jpg'
audo_path = 'jr://file/commcare/case_list_audo.mp3'
app.get_module(0).case_list_form.media_image = image_path
app.get_module(0).case_list_form.media_audio = audo_path
app.create_mapping(CommCareImage(_id='123'), image_path, save=False)
app.create_mapping(CommCareAudio(_id='456'), audo_path, save=False)
app.set_media_versions(previous_version=None)
self.assertXmlEqual(self.get_xml('media_suite'), app.create_media_suite())
def match_file(self, uploaded_file, replace_existing_media=True, **kwargs):
errors = []
try:
if self.specific_params and self.specific_params['media_type'][0].startswith('CommCare'):
media_class = getattr(sys.modules[__name__], self.specific_params['media_type'][0])
form_path = self.specific_params['path'][0]
replace_existing_media = self.specific_params['replace_attachment'][0]
filename = uploaded_file.name
data = uploaded_file.file.read()
media = media_class.get_by_data(data)
else:
filename = uploaded_file.name
if filename in self.images:
form_path = self.image_paths[self.images.index(filename)]
data = uploaded_file.file.read()
media = CommCareImage.get_by_data(data)
elif filename in self.audio:
form_path = self.audio_paths[self.audio.index(filename)]
data = uploaded_file.file.read()
media = CommCareAudio.get_by_data(data)
else:
uploaded_file.close()
return False, {}, errors
if media:
media.attach_data(data,
original_filename=filename,
username=self.username,
replace_attachment=replace_existing_media)
media.add_domain(self.domain, owner=True, **kwargs)
media.update_or_add_license(self.domain,
type=kwargs.get('license', ''),
author=kwargs.get('author', ''),
attribution_notes=kwargs.get('attribution_notes', ''))
self.app.create_mapping(media, form_path)
return True, HQMediaMapItem.format_match_map(form_path, media.doc_type, media._id, filename), errors
except Exception as e:
logging.error(e)
errors.append(e.message)
return False, {}, errors
def test_media_suite_generator(self):
app = Application.wrap(self.get_json('app_video_inline'))
image_path = 'jr://file/commcare/image1.jpg'
audio_path = 'jr://file/commcare/audio1.mp3'
video_path = 'jr://file/commcare/video-inline/data/inline_video.mp4'
app.create_mapping(CommCareImage(_id='123'), image_path, save=False)
app.create_mapping(CommCareAudio(_id='456'), audio_path, save=False)
app.create_mapping(CommCareVideo(_id='789'), video_path, save=False)
app.get_module(0).case_list_form.set_icon('en', image_path)
app.get_module(0).case_list_form.set_audio('en', audio_path)
app.get_module(0).case_list_form.form_id = app.get_module(0).get_form(
0).unique_id
app.profile["properties"] = {'lazy-load-video-files': 'true'}
self.assertXmlEqual(self.get_xml('media-suite-lazy-true'),
MediaSuiteGenerator(app).generate_suite())
app.profile["properties"] = {'lazy-load-video-files': 'false'}
self.assertXmlEqual(self.get_xml('media-suite-lazy-false'),
MediaSuiteGenerator(app).generate_suite())
def search_for_media(request, domain, app_id):
media_type = request.GET["t"]
if media_type == "Image":
files = CommCareImage.search(request.GET["q"])
elif media_type == "Audio":
files = CommCareAudio.search(request.GET["q"])
else:
raise Http404()
return HttpResponse(
json.dumps(
[
{
"url": i.url(),
"licenses": [license.display_name for license in i.licenses],
"tags": [tag for tags in i.tags.values() for tag in tags],
"m_id": i._id,
}
for i in files
]
)
)
def _get_app(self):
factory = AppFactory(build_version='2.40.0')
app = factory.app
app.langs = ['en', 'fra']
module, form = factory.new_basic_module('register', 'case')
'''
This form has two text questions, one with French audio present and one with it missing.
It then has three select questions, each with two choices that share text and all
have French audio specified. One set of choices has audio present for both, one has
it missing for both, and one has it present for one choice and missing for the other.
Lastly, it has two select questions, each with three choices that all have different
text but use the same audio path. One of these two audio files is missing.
'''
form.source = self.get_xml('duplicate_text_questions').decode('utf-8')
for path in form.wrapped_xform().audio_references():
if 'present' in path or 'one_of_each1' in path:
app.create_mapping(CommCareAudio(_id=uuid.uuid4().hex), path, save=False)
return app
def test_multi_master_form_attributes_and_media_versions(self, *args):
'''
This tests a few things related to pulling a linked app from multiple master apps,
particularly interleaving pulls (pulling master A, then master B, then master A again):
- Form versions should not change unless the form content changed
- Form unique ids should be different from the master they came from but consistent across
versions of the linked app that come from that master.
- If a new form is added to multiple masters, that form's unique id should be the same
across all versions of the linked app that pull from any of those masters - that is,
each XMLNS in the linked app should correspond to one and only one form unique id.
- Multimedia versions should not change, but should be consistent with the version
of the linked app where they were introduced.
'''
# Add single module and form to both master1 and master2.
# The module in master1 will also be used for multimedia testing.
master1_module = self.master1.add_module(
Module.new_module('Module for master1', None))
master1_module.new_form('Form for master1', 'en',
get_blank_form_xml('Form for master1'))
master1_map = self._get_form_ids_by_xmlns(self.master1)
image_path = 'jr://file/commcare/photo.jpg'
self.master1.create_mapping(CommCareImage(_id='123'), image_path)
self.master1.get_module(0).set_icon('en', image_path)
self._make_master1_build(True)
master2_module = self.master2.add_module(
Module.new_module('Module for master2', None))
master2_module.new_form('Form for master2', 'en',
get_blank_form_xml('Form for master2'))
master2_map = self._get_form_ids_by_xmlns(self.master2)
self._make_master2_build(True)
# Pull master1, so linked app now has a form. Verify that form xmlnses match.
self._pull_linked_app(self.master1.get_id)
linked_master1_build1 = self._make_linked_build()
linked_master1_build1_form = linked_master1_build1.get_module(
0).get_form(0)
linked_master1_map = self._get_form_ids_by_xmlns(self.linked_app)
self.assertEqual(master1_map, linked_master1_map)
original_image_version = linked_master1_build1.multimedia_map[
image_path].version
self.assertEqual(original_image_version, linked_master1_build1.version)
# Pull master2, so linked app now has other form. Verify that form xmlnses match but unique ids do not.
self._pull_linked_app(self.master2.get_id)
linked_master2_build1 = self._make_linked_build()
linked_master2_map = self._get_form_ids_by_xmlns(self.linked_app)
linked_master2_build1_form = linked_master2_build1.get_module(
0).get_form(0)
self.assertEqual(master2_map, linked_master2_map)
self.assertNotEqual(linked_master1_build1_form.unique_id,
linked_master2_build1_form.unique_id)
# Re-pull master1, so linked app is back to the first form, with same xmlns, unique id, and version
linked_master1_build1 = self._make_master1_build(True)
self._pull_linked_app(self.master1.get_id)
linked_master1_build2 = self._make_linked_build()
linked_master1_build2_form = linked_master1_build2.get_module(
0).get_form(0)
self.assertEqual(linked_master1_map,
self._get_form_ids_by_xmlns(self.linked_app))
self.assertEqual(linked_master1_build1_form.xmlns,
linked_master1_build2_form.xmlns)
self.assertEqual(linked_master1_build1_form.unique_id,
linked_master1_build2_form.unique_id)
self.assertEqual(linked_master1_build1_form.get_version(),
linked_master1_build2_form.get_version())
# Update form in master1 and make new linked build, which should update form version
# Also add audio. The new audio should get the new build version, but the old image should retain
# the version of the old app.
wrapped = self.master1.get_module(0).get_form(0).wrapped_xform()
wrapped.set_name("Updated form for master1")
self.master1.get_module(0).get_form(0).source = etree.tostring(
wrapped.xml, encoding='utf-8')
audio_path = 'jr://file/commcare/scream.mp3'
self.master1.create_mapping(CommCareAudio(_id='345'), audio_path)
self.master1.get_module(0).set_audio('en', audio_path)
self._make_master1_build(True)
self._pull_linked_app(self.master1.get_id)
linked_master1_build3 = self._make_linked_build()
linked_master1_build3_form = linked_master1_build3.get_module(
0).get_form(0)
self.assertEqual(linked_master1_build2_form.xmlns,
linked_master1_build3_form.xmlns)
self.assertEqual(linked_master1_build2_form.unique_id,
linked_master1_build3_form.unique_id)
self.assertLess(linked_master1_build2_form.get_version(),
linked_master1_build3_form.get_version())
self.assertEqual(self.linked_app.multimedia_map[image_path].version,
original_image_version)
self.assertGreater(self.linked_app.multimedia_map[audio_path].version,
original_image_version)
# Add another form to both master1 and master2. When master1 is pulled, that form should be assigned a
# new unique id, and when master2 is pulled, it should retain that id since it has the same xmlns.
self.master1.get_module(0).new_form(
'Twin form', None,
self.get_xml('very_simple_form').decode('utf-8'))
self._make_master1_build(True)
self._pull_linked_app(self.master1.get_id)
xmlns = self.master1.get_module(0).get_form(1).xmlns
self.master2.get_module(0).new_form(
'Twin form', None,
self.get_xml('very_simple_form').decode('utf-8'))
linked_master1_build4 = self._make_linked_build()
self._make_master2_build(True)
self._pull_linked_app(self.master2.get_id)
linked_master2_build2 = self._make_linked_build()
self.assertEqual(xmlns, self.master2.get_module(0).get_form(1).xmlns)
self.assertEqual(
self._get_form_ids_by_xmlns(linked_master1_build4)[xmlns],
self._get_form_ids_by_xmlns(self.master1)[xmlns])
self.assertEqual(
self._get_form_ids_by_xmlns(linked_master2_build2)[xmlns],
self._get_form_ids_by_xmlns(self.master2)[xmlns])
