def test_multimedia(self): from corehq.apps.hqmedia.models import CommCareAudio, CommCareImage, CommCareVideo image_path = os.path.join('corehq', 'apps', 'hqwebapp', 'static', 'hqwebapp', 'images', 'commcare-hq-logo.png') with open(image_path, 'r') as f: image_data = f.read() image = CommCareImage.get_by_data(image_data) image.attach_data(image_data, original_filename='logo.png') image.add_domain(self.domain_name) self.assertEqual(image_data, image.get_display_file(False)) audio_data = 'fake audio data' audio = CommCareAudio.get_by_data(audio_data) audio.attach_data(audio_data, original_filename='tr-la-la.mp3') audio.add_domain(self.domain_name) self.assertEqual(audio_data, audio.get_display_file(False)) video_data = 'fake video data' video = CommCareVideo.get_by_data(video_data) video.attach_data(video_data, 'kittens.mp4') video.add_domain(self.domain_name) self.assertEqual(video_data, video.get_display_file(False)) fakedb = self._dump_and_load([image, audio, video]) copied_image = CommCareImage.wrap(fakedb.get(image._id)) copied_audio = CommCareAudio.wrap(fakedb.get(audio._id)) copied_video = CommCareVideo.wrap(fakedb.get(video._id)) self.assertEqual(image_data, copied_image.get_display_file(False)) self.assertEqual(audio_data, copied_audio.get_display_file(False)) self.assertEqual(video_data, copied_video.get_display_file(False))
def test_multimedia(self): from corehq.apps.hqmedia.models import CommCareAudio, CommCareImage, CommCareVideo image_path = os.path.join('corehq', 'apps', 'hqwebapp', 'static', 'hqwebapp', 'img', 'commcare-hq-logo.png') with open(image_path, 'r') as f: image_data = f.read() image = CommCareImage.get_by_data(image_data) image.attach_data(image_data, original_filename='logo.png') image.add_domain(self.domain_name) self.assertEqual(image_data, image.get_display_file(False)) audio_data = 'fake audio data' audio = CommCareAudio.get_by_data(audio_data) audio.attach_data(audio_data, original_filename='tr-la-la.mp3') audio.add_domain(self.domain_name) self.assertEqual(audio_data, audio.get_display_file(False)) video_data = 'fake video data' video = CommCareVideo.get_by_data(video_data) video.attach_data(video_data, 'kittens.mp4') video.add_domain(self.domain_name) self.assertEqual(video_data, video.get_display_file(False)) fakedb = self._dump_and_load([image, audio, video]) copied_image = CommCareImage.wrap(fakedb.get(image._id)) copied_audio = CommCareAudio.wrap(fakedb.get(audio._id)) copied_video = CommCareVideo.wrap(fakedb.get(video._id)) self.assertEqual(image_data, copied_image.get_display_file(False)) self.assertEqual(audio_data, copied_audio.get_display_file(False)) self.assertEqual(video_data, copied_video.get_display_file(False))
def test_form_media_with_app_profile(self, *args): # Test that media for languages not in the profile are removed from the media suite app = Application.wrap(self.get_json('app')) app.build_profiles['profid'] = BuildProfile(langs=['en'], name='en-profile') app.langs = ['en', 'hin'] image_path = 'jr://file/commcare/module0_en.png' audio_path = 'jr://file/commcare/module0_{}.mp3' app.get_module(0).set_icon('en', image_path) # app.get_module(0).case_list_form.set_icon('en', image_path) app.get_module(0).set_audio('en', audio_path.format('en')) app.get_module(0).set_audio('hin', audio_path.format('hin')) app.create_mapping(CommCareImage(_id='123'), image_path, save=False) app.create_mapping(CommCareAudio(_id='456'), audio_path.format('en'), save=False) app.create_mapping(CommCareAudio(_id='789'), audio_path.format('hin'), save=False) form_xml = self.get_xml('form_with_media_refs').decode('utf-8') form = app.get_module(0).new_form('form_with_media', 'en', attachment=form_xml) xform = form.wrapped_xform() for i, path in enumerate(xform.media_references(form="audio")): app.create_mapping(CommCareAudio(_id='form_audio_{}'.format(i)), path, save=False) for i, path in enumerate(xform.media_references(form="image")): app.create_mapping(CommCareImage(_id='form_image_{}'.format(i)), path, save=False) app.set_media_versions() app.remove_unused_mappings() # includes all media self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite'), app.create_media_suite()) # includes all app media and only form media for 'en' self._assertMediaSuiteResourcesEqual( self.get_xml('form_media_suite_en'), app.create_media_suite('profid'))
def test_unused_media_removed(self, mock): image_path = 'jr://file/commcare/image{}_{}.jpg' audio_path = 'jr://file/commcare/audio{}_{}.mp3' app = Application.wrap(self.get_json('app')) app.domain = self.domain app.save() for lang in ['en', 'hin']: for num in ['1', '2', '3', '4']: app.create_mapping(CommCareImage(_id=num), image_path.format(num, lang), save=False) app.create_mapping(CommCareAudio(_id=num), audio_path.format(num, lang), save=False) app.get_module(0).case_list.show = True app.get_module(0).case_list.set_icon('en', image_path.format('4', 'en')) app.get_module(0).case_list.set_audio('en', audio_path.format('4', 'en')) app.get_module(0).set_icon('en', image_path.format('1', 'en')) app.get_module(0).set_audio('en', audio_path.format('1', 'en')) app.get_module(0).case_list_form.form_id = app.get_module(0).get_form( 0).unique_id app.get_module(0).case_list_form.set_icon('en', image_path.format('2', 'en')) app.get_module(0).case_list_form.set_audio( 'en', audio_path.format('2', 'en')) app.get_module(0).get_form(0).set_icon('en', image_path.format('3', 'en')) app.get_module(0).get_form(0).set_audio('en', audio_path.format('3', 'en')) app.save() should_contain_media = [image_path.format(num, 'en') for num in [1, 2, 3, 4]] + \ [audio_path.format(num, 'en') for num in [1, 2, 3, 4]] media_for_removal = [image_path.format(num, 'hin') for num in [1, 2, 3, 4]] + \ [audio_path.format(num, 'hin') for num in [1, 2, 3, 4]] self.assertTrue(app.get_module(0).uses_media()) self.assertEqual(app.all_media_paths(), set(should_contain_media)) self.assertEqual(set(app.multimedia_map.keys()), set(should_contain_media + media_for_removal)) app.remove_unused_mappings() self.assertEqual(set(app.multimedia_map.keys()), set(should_contain_media)) # test multimedia removed app.get_module(0).case_list.set_icon('en', '') app.get_module(0).case_list.set_audio('en', '') app.get_module(0).set_icon('en', '') app.get_module(0).set_audio('en', '') app.get_module(0).case_list_form.set_icon('en', '') app.get_module(0).case_list_form.set_audio('en', '') app.get_module(0).get_form(0).set_icon('en', '') app.get_module(0).get_form(0).set_audio('en', '') self.assertFalse(list(app.multimedia_map.keys()))
def match_zipped(self, zip, replace_existing_media=True, **kwargs): unknown_files = [] matched_audio = [] matched_images = [] errors = [] try: for index, path in enumerate(zip.namelist()): path = path.strip() filename = path.lower() basename = os.path.basename(path.lower()) is_image = False if not basename: continue if self.match_filename: filename = basename media = None form_path = None data = None if filename in self.images: form_path = self.image_paths[self.images.index(filename)] data = zip.read(path) media = CommCareImage.get_by_data(data) is_image = True elif filename in self.audio: form_path = self.audio_paths[self.audio.index(filename)] data = zip.read(path) media = CommCareAudio.get_by_data(data) else: unknown_files.append(path) if media: try: media.attach_data(data, upload_path=path, username=self.username, replace_attachment=replace_existing_media) media.add_domain(self.domain, owner=True) media.update_or_add_license(self.domain, type=kwargs.get('license', ''), author=kwargs.get('author', ''), attribution_notes=kwargs.get('attribution_notes', '')) self.app.create_mapping(media, form_path) match_map = HQMediaMapItem.format_match_map(form_path, media.doc_type, media._id, path) if is_image: matched_images.append(match_map) else: matched_audio.append(match_map) except Exception as e: errors.append("%s (%s)" % (e, filename)) zip.close() except Exception as e: logging.error(e) errors.append(e.message) return matched_images, matched_audio, unknown_files, errors
def test_form_media_with_app_profile(self, *args): # Test that media for languages not in the profile are removed from the media suite app = Application.wrap(self.get_json('app')) app.build_profiles = OrderedDict({ 'en': BuildProfile(langs=['en'], name='en-profile'), 'hin': BuildProfile(langs=['hin'], name='hin-profile'), 'all': BuildProfile(langs=['en', 'hin'], name='all-profile'), }) app.langs = ['en', 'hin'] image_path = 'jr://file/commcare/module0_en.png' audio_path = 'jr://file/commcare/module0_{}.mp3' app.get_module(0).set_icon('en', image_path) app.get_module(0).set_audio('en', audio_path.format('en')) app.get_module(0).set_audio('hin', audio_path.format('hin')) app.create_mapping(CommCareImage(_id='123'), image_path, save=False) app.create_mapping(CommCareAudio(_id='456'), audio_path.format('en'), save=False) app.create_mapping(CommCareAudio(_id='789'), audio_path.format('hin'), save=False) form_xml = self.get_xml('form_with_media_refs').decode('utf-8') form = app.get_module(0).new_form('form_with_media', 'en', attachment=form_xml) xform = form.wrapped_xform() for i, path in enumerate(reversed(sorted(xform.media_references(form="audio")))): app.create_mapping(CommCareAudio(_id='form_audio_{}'.format(i)), path, save=False) for i, path in enumerate(sorted(xform.media_references(form="image"))): app.create_mapping(CommCareImage(_id='form_image_{}'.format(i)), path, save=False) app.set_media_versions() app.remove_unused_mappings() # includes all media self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite'), app.create_media_suite()) # generate all suites at once to mimic create_build_files_for_all_app_profiles suites = {id: app.create_media_suite(build_profile_id=id) for id in app.build_profiles.keys()} # include all app media and only language-specific form media self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite_en'), suites['en']) self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite_hin'), suites['hin']) self._assertMediaSuiteResourcesEqual(self.get_xml('form_media_suite_all'), suites['all'])
def save(self, domain, app, username, cache_handler=None): orig_images, orig_audio, _ = utils.get_multimedia_filenames(app) form_images = [i.replace(utils.MULTIMEDIA_PREFIX, '').lower().strip() for i in orig_images] form_audio = [a.replace(utils.MULTIMEDIA_PREFIX, '').lower().strip() for a in orig_audio] if self.cleaned_data["repopulate_multimedia_map"]: app.multimedia_map = {} app.save() replace_attachment = self.cleaned_data["replace_existing_media"] zip = self.cleaned_data["zip_file"] num_files = len(zip.namelist()) if cache_handler: cache_handler.sync() cache_handler.data["processed_length"] = num_files cache_handler.save() unknown_files = [] for index, path in enumerate(zip.namelist()): path = path.strip() path_lower = path.lower() if path_lower.endswith("/") or path_lower.endswith("\\") or \ path_lower.endswith(".ds_store"): continue media = None form_path = None data = None if path_lower in form_images: form_path = orig_images[form_images.index(path_lower)] data = zip.read(path) media = CommCareImage.get_by_data(data) elif path_lower in form_audio: form_path = orig_audio[form_audio.index(path_lower)] data = zip.read(path) media = CommCareAudio.get_by_data(data) else: unknown_files.append(path) if media: media.attach_data(data, upload_path=path, username=username, replace_attachment=replace_attachment) media.add_domain(domain, owner=True) app.create_mapping(media, form_path) if cache_handler: cache_handler.sync() cache_handler.data["processed"] = index+1 cache_handler.save() zip.close() return {"successful": unknown_files}
def search_for_media(request, domain, app_id): media_type = request.GET['t'] if media_type == 'Image': files = CommCareImage.search(request.GET['q']) elif media_type == 'Audio': files = CommCareAudio.search(request.GET['q']) else: raise Http404() return HttpResponse(json.dumps([ {'url': i.url(), 'licenses': [license.display_name for license in i.licenses], 'tags': [tag for tags in i.tags.values() for tag in tags], 'm_id': i._id} for i in files]))
def test_case_list_media(self): app = Application.wrap(self.get_json('app')) app.get_module(0).case_list_form.form_id = app.get_module(0).get_form(0).unique_id image_path = 'jr://file/commcare/case_list_image.jpg' audo_path = 'jr://file/commcare/case_list_audo.mp3' app.get_module(0).case_list_form.set_icon('en', image_path) app.get_module(0).case_list_form.set_audio('en', audo_path) app.create_mapping(CommCareImage(_id='123'), image_path, save=False) app.create_mapping(CommCareAudio(_id='456'), audo_path, save=False) app.set_media_versions() self._assertMediaSuiteResourcesEqual(self.get_xml('case_list_media_suite'), app.create_media_suite())
def test_case_list_media(self): app = Application.wrap(self.get_json('app')) app.get_module(0).case_list_form.form_id = app.get_module(0).get_form(0).unique_id image_path = 'jr://file/commcare/case_list_image.jpg' audo_path = 'jr://file/commcare/case_list_audo.mp3' app.get_module(0).case_list_form.media_image = image_path app.get_module(0).case_list_form.media_audio = audo_path app.create_mapping(CommCareImage(_id='123'), image_path, save=False) app.create_mapping(CommCareAudio(_id='456'), audo_path, save=False) app.set_media_versions(previous_version=None) self.assertXmlEqual(self.get_xml('media_suite'), app.create_media_suite())
def match_file(self, uploaded_file, replace_existing_media=True, **kwargs): errors = [] try: if self.specific_params and self.specific_params['media_type'][0].startswith('CommCare'): media_class = getattr(sys.modules[__name__], self.specific_params['media_type'][0]) form_path = self.specific_params['path'][0] replace_existing_media = self.specific_params['replace_attachment'][0] filename = uploaded_file.name data = uploaded_file.file.read() media = media_class.get_by_data(data) else: filename = uploaded_file.name if filename in self.images: form_path = self.image_paths[self.images.index(filename)] data = uploaded_file.file.read() media = CommCareImage.get_by_data(data) elif filename in self.audio: form_path = self.audio_paths[self.audio.index(filename)] data = uploaded_file.file.read() media = CommCareAudio.get_by_data(data) else: uploaded_file.close() return False, {}, errors if media: media.attach_data(data, original_filename=filename, username=self.username, replace_attachment=replace_existing_media) media.add_domain(self.domain, owner=True, **kwargs) media.update_or_add_license(self.domain, type=kwargs.get('license', ''), author=kwargs.get('author', ''), attribution_notes=kwargs.get('attribution_notes', '')) self.app.create_mapping(media, form_path) return True, HQMediaMapItem.format_match_map(form_path, media.doc_type, media._id, filename), errors except Exception as e: logging.error(e) errors.append(e.message) return False, {}, errors
def test_media_suite_generator(self): app = Application.wrap(self.get_json('app_video_inline')) image_path = 'jr://file/commcare/image1.jpg' audio_path = 'jr://file/commcare/audio1.mp3' video_path = 'jr://file/commcare/video-inline/data/inline_video.mp4' app.create_mapping(CommCareImage(_id='123'), image_path, save=False) app.create_mapping(CommCareAudio(_id='456'), audio_path, save=False) app.create_mapping(CommCareVideo(_id='789'), video_path, save=False) app.get_module(0).case_list_form.set_icon('en', image_path) app.get_module(0).case_list_form.set_audio('en', audio_path) app.get_module(0).case_list_form.form_id = app.get_module(0).get_form( 0).unique_id app.profile["properties"] = {'lazy-load-video-files': 'true'} self.assertXmlEqual(self.get_xml('media-suite-lazy-true'), MediaSuiteGenerator(app).generate_suite()) app.profile["properties"] = {'lazy-load-video-files': 'false'} self.assertXmlEqual(self.get_xml('media-suite-lazy-false'), MediaSuiteGenerator(app).generate_suite())
def search_for_media(request, domain, app_id): media_type = request.GET["t"] if media_type == "Image": files = CommCareImage.search(request.GET["q"]) elif media_type == "Audio": files = CommCareAudio.search(request.GET["q"]) else: raise Http404() return HttpResponse( json.dumps( [ { "url": i.url(), "licenses": [license.display_name for license in i.licenses], "tags": [tag for tags in i.tags.values() for tag in tags], "m_id": i._id, } for i in files ] ) )
def _get_app(self): factory = AppFactory(build_version='2.40.0') app = factory.app app.langs = ['en', 'fra'] module, form = factory.new_basic_module('register', 'case') ''' This form has two text questions, one with French audio present and one with it missing. It then has three select questions, each with two choices that share text and all have French audio specified. One set of choices has audio present for both, one has it missing for both, and one has it present for one choice and missing for the other. Lastly, it has two select questions, each with three choices that all have different text but use the same audio path. One of these two audio files is missing. ''' form.source = self.get_xml('duplicate_text_questions').decode('utf-8') for path in form.wrapped_xform().audio_references(): if 'present' in path or 'one_of_each1' in path: app.create_mapping(CommCareAudio(_id=uuid.uuid4().hex), path, save=False) return app
def test_multi_master_form_attributes_and_media_versions(self, *args): ''' This tests a few things related to pulling a linked app from multiple master apps, particularly interleaving pulls (pulling master A, then master B, then master A again): - Form versions should not change unless the form content changed - Form unique ids should be different from the master they came from but consistent across versions of the linked app that come from that master. - If a new form is added to multiple masters, that form's unique id should be the same across all versions of the linked app that pull from any of those masters - that is, each XMLNS in the linked app should correspond to one and only one form unique id. - Multimedia versions should not change, but should be consistent with the version of the linked app where they were introduced. ''' # Add single module and form to both master1 and master2. # The module in master1 will also be used for multimedia testing. master1_module = self.master1.add_module( Module.new_module('Module for master1', None)) master1_module.new_form('Form for master1', 'en', get_blank_form_xml('Form for master1')) master1_map = self._get_form_ids_by_xmlns(self.master1) image_path = 'jr://file/commcare/photo.jpg' self.master1.create_mapping(CommCareImage(_id='123'), image_path) self.master1.get_module(0).set_icon('en', image_path) self._make_master1_build(True) master2_module = self.master2.add_module( Module.new_module('Module for master2', None)) master2_module.new_form('Form for master2', 'en', get_blank_form_xml('Form for master2')) master2_map = self._get_form_ids_by_xmlns(self.master2) self._make_master2_build(True) # Pull master1, so linked app now has a form. Verify that form xmlnses match. self._pull_linked_app(self.master1.get_id) linked_master1_build1 = self._make_linked_build() linked_master1_build1_form = linked_master1_build1.get_module( 0).get_form(0) linked_master1_map = self._get_form_ids_by_xmlns(self.linked_app) self.assertEqual(master1_map, linked_master1_map) original_image_version = linked_master1_build1.multimedia_map[ image_path].version self.assertEqual(original_image_version, linked_master1_build1.version) # Pull master2, so linked app now has other form. Verify that form xmlnses match but unique ids do not. self._pull_linked_app(self.master2.get_id) linked_master2_build1 = self._make_linked_build() linked_master2_map = self._get_form_ids_by_xmlns(self.linked_app) linked_master2_build1_form = linked_master2_build1.get_module( 0).get_form(0) self.assertEqual(master2_map, linked_master2_map) self.assertNotEqual(linked_master1_build1_form.unique_id, linked_master2_build1_form.unique_id) # Re-pull master1, so linked app is back to the first form, with same xmlns, unique id, and version linked_master1_build1 = self._make_master1_build(True) self._pull_linked_app(self.master1.get_id) linked_master1_build2 = self._make_linked_build() linked_master1_build2_form = linked_master1_build2.get_module( 0).get_form(0) self.assertEqual(linked_master1_map, self._get_form_ids_by_xmlns(self.linked_app)) self.assertEqual(linked_master1_build1_form.xmlns, linked_master1_build2_form.xmlns) self.assertEqual(linked_master1_build1_form.unique_id, linked_master1_build2_form.unique_id) self.assertEqual(linked_master1_build1_form.get_version(), linked_master1_build2_form.get_version()) # Update form in master1 and make new linked build, which should update form version # Also add audio. The new audio should get the new build version, but the old image should retain # the version of the old app. wrapped = self.master1.get_module(0).get_form(0).wrapped_xform() wrapped.set_name("Updated form for master1") self.master1.get_module(0).get_form(0).source = etree.tostring( wrapped.xml, encoding='utf-8') audio_path = 'jr://file/commcare/scream.mp3' self.master1.create_mapping(CommCareAudio(_id='345'), audio_path) self.master1.get_module(0).set_audio('en', audio_path) self._make_master1_build(True) self._pull_linked_app(self.master1.get_id) linked_master1_build3 = self._make_linked_build() linked_master1_build3_form = linked_master1_build3.get_module( 0).get_form(0) self.assertEqual(linked_master1_build2_form.xmlns, linked_master1_build3_form.xmlns) self.assertEqual(linked_master1_build2_form.unique_id, linked_master1_build3_form.unique_id) self.assertLess(linked_master1_build2_form.get_version(), linked_master1_build3_form.get_version()) self.assertEqual(self.linked_app.multimedia_map[image_path].version, original_image_version) self.assertGreater(self.linked_app.multimedia_map[audio_path].version, original_image_version) # Add another form to both master1 and master2. When master1 is pulled, that form should be assigned a # new unique id, and when master2 is pulled, it should retain that id since it has the same xmlns. self.master1.get_module(0).new_form( 'Twin form', None, self.get_xml('very_simple_form').decode('utf-8')) self._make_master1_build(True) self._pull_linked_app(self.master1.get_id) xmlns = self.master1.get_module(0).get_form(1).xmlns self.master2.get_module(0).new_form( 'Twin form', None, self.get_xml('very_simple_form').decode('utf-8')) linked_master1_build4 = self._make_linked_build() self._make_master2_build(True) self._pull_linked_app(self.master2.get_id) linked_master2_build2 = self._make_linked_build() self.assertEqual(xmlns, self.master2.get_module(0).get_form(1).xmlns) self.assertEqual( self._get_form_ids_by_xmlns(linked_master1_build4)[xmlns], self._get_form_ids_by_xmlns(self.master1)[xmlns]) self.assertEqual( self._get_form_ids_by_xmlns(linked_master2_build2)[xmlns], self._get_form_ids_by_xmlns(self.master2)[xmlns])