def page_context(self): context = super(MultimediaTranslationsCoverageView, self).page_context selected_build_id = self.request.POST.get('build_id') selected_build_text = '' if selected_build_id: build = get_app(self.app.domain, selected_build_id) selected_build_text = str(build.version) if build.build_comment: selected_build_text += ": " + build.build_comment context.update({ "media_types": { t: CommCareMultimedia.get_doc_class(t).get_nice_name() for t in CommCareMultimedia.get_doc_types() }, "selected_langs": self.request.POST.getlist('langs', []), "selected_media_types": self.request.POST.getlist('media_types', ['CommCareAudio', 'CommCareVideo']), "selected_build_id": selected_build_id, "selected_build_text": selected_build_text, }) return context
def app_from_template(request, domain, slug): send_hubspot_form(HUBSPOT_APP_TEMPLATE_FORM_ID, request) clear_app_cache(request, domain) template = load_app_template(slug) app = import_app_util(template, domain, { 'created_from_template': '%s' % slug, }) for path in get_template_app_multimedia_paths(slug): media_class = None with open(os.path.join(app_template_dir(slug), path), "rb") as f: f.seek(0) data = f.read() media_class = CommCareMultimedia.get_class_by_data(data) if media_class: multimedia = media_class.get_by_data(data) multimedia.attach_data( data, original_filename=os.path.basename(path), username=request.user.username) multimedia.add_domain(domain, owner=True) app.create_mapping(multimedia, MULTIMEDIA_PREFIX + path) comment = _("A sample application you can try out in Web Apps") build = make_async_build(app, request.user.username, release=True, comment=comment) cloudcare_state = '{{"appId":"{}"}}'.format(build._id) return HttpResponseRedirect( reverse(FormplayerMain.urlname, args=[domain]) + '#' + cloudcare_state)
def load_app_from_slug(domain, username, slug): # Import app itself template = load_app_template(slug) app = import_app_util(template, domain, { 'created_from_template': '%s' % slug, }) # Fetch multimedia, which is hosted elsewhere multimedia_filename = os.path.join(app_template_dir(slug), 'multimedia.json') if (os.path.exists(multimedia_filename)): with open(multimedia_filename) as f: path_url_map = json.load(f) http = urllib3.PoolManager() for path, url in path_url_map.items(): try: req = http.request('GET', url) except Exception: # If anything goes wrong, just bail. It's not a big deal if a template app is missing a file. continue if req.status == 200: data = req.data media_class = CommCareMultimedia.get_class_by_data(data) if media_class: multimedia = media_class.get_by_data(data) multimedia.attach_data(data, original_filename=os.path.basename(path), username=username) multimedia.add_domain(domain, owner=True) app.create_mapping(multimedia, MULTIMEDIA_PREFIX + path) comment = _("A sample application you can try out in Web Apps") build = make_async_build(app, username, allow_prune=False, comment=comment) build.is_released = True build.save(increment_version=False) return build
def all_media(self, from_apps=None): # todo add documentation or refactor from corehq.apps.hqmedia.models import CommCareMultimedia dom_with_media = self if not self.is_snapshot else self.copied_from if self.is_snapshot: app_ids = [ app.copied_from.get_id for app in self.full_applications() ] if from_apps: from_apps = set( [a_id for a_id in app_ids if a_id in from_apps]) else: from_apps = app_ids if from_apps: media = [] media_ids = set() apps = [ app for app in dom_with_media.full_applications() if app.get_id in from_apps ] for app in apps: if app.doc_type != 'Application': continue for _, m in app.get_media_objects(): if m.get_id not in media_ids: media.append(m) media_ids.add(m.get_id) return media return CommCareMultimedia.view('hqmedia/by_domain', key=dom_with_media.name, include_docs=True).all()
def _get_couchdb_counts(domain): couch_db_counts = Counter() for provider in DOC_PROVIDERS: if isinstance(provider, DocTypeIDProvider): for doc_type in provider.doc_types: if doc_type == 'CommCareUser': continue # want to split deleted doc_class = get_document_class_by_doc_type(doc_type) count = get_doc_count_in_domain_by_class(domain, doc_class) couch_db_counts.update({doc_type: count}) for row in CommCareMultimedia.get_db().view('hqmedia/by_domain', key=domain, include_docs=False): couch_db_counts.update(['CommCareMultimedia']) mobile_user_count = get_mobile_user_count(domain) couch_db_counts.update({ 'WebUser': get_web_user_count(domain), 'CommCareUser': mobile_user_count, 'CommCareUser-Deleted': get_doc_count_in_domain_by_class(domain, CommCareUser) - mobile_user_count }) # this is very slow, excluding for now # for _, doc_ids in SyncLogIDProvider().get_doc_ids(domain): # couch_db_counts['SyncLog'] += len(doc_ids) # return couch_db_counts
def test_fetch_missing_media(self): image_path = 'jr://file/commcare/case_list_image.jpg' self.master_app_with_report_modules.get_module(0).set_icon( 'en', image_path) self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False) remote_details = RemoteLinkDetails('http://localhost:8000', 'user', 'key') data = b'this is a test: \255' # Real data will be a binary multimedia file, so mock it with bytes, not unicode media_details = list( self.master_app_with_report_modules.multimedia_map.values())[0] media_details['multimedia_id'] = uuid.uuid4().hex media_details['media_type'] = 'CommCareMultimedia' with patch( 'corehq.apps.linked_domain.remote_accessors._fetch_remote_media_content' ) as mock: mock.return_value = data fetch_remote_media('domain', [('case_list_image.jpg', media_details)], remote_details) media = CommCareMultimedia.get(media_details['multimedia_id']) self.addCleanup(media.delete) content = media.fetch_attachment(list(media.blobs.keys())[0]) self.assertEqual(data, content)
def load_app_from_slug(domain, username, slug): # Import app itself template = load_app_template(slug) app = import_app_util(template, domain, { 'created_from_template': '%s' % slug, }) # Fetch multimedia, which is hosted elsewhere multimedia_filename = os.path.join(app_template_dir(slug), 'multimedia.json') if (os.path.exists(multimedia_filename)): with open(multimedia_filename) as f: path_url_map = json.load(f) http = urllib3.PoolManager() for path, url in path_url_map.items(): try: req = http.request('GET', url) except Exception: # If anything goes wrong, just bail. It's not a big deal if a template app is missing a file. continue if req.status == 200: data = req.data media_class = CommCareMultimedia.get_class_by_data(data) if media_class: multimedia = media_class.get_by_data(data) multimedia.attach_data(data, original_filename=os.path.basename(path), username=username) multimedia.add_domain(domain, owner=True) app.create_mapping(multimedia, MULTIMEDIA_PREFIX + path) return _build_sample_app(app)
def all_media(self, from_apps=None): #todo add documentation or refactor from corehq.apps.hqmedia.models import CommCareMultimedia dom_with_media = self if not self.is_snapshot else self.copied_from if self.is_snapshot: app_ids = [app.copied_from.get_id for app in self.full_applications()] if from_apps: from_apps = set([a_id for a_id in app_ids if a_id in from_apps]) else: from_apps = app_ids if from_apps: media = [] media_ids = set() apps = [app for app in dom_with_media.full_applications() if app.get_id in from_apps] for app in apps: if app.doc_type != 'Application': continue for _, m in app.get_media_objects(): if m.get_id not in media_ids: media.append(m) media_ids.add(m.get_id) return media return CommCareMultimedia.view('hqmedia/by_domain', key=dom_with_media.name, include_docs=True).all()
def save(self, domain, app, username, cache_handler): media_class = CommCareMultimedia.get_doc_class(self.cleaned_data["multimedia_class"]) media_file = self.cleaned_data["media_file"] replace_attachment = self.cleaned_data["multimedia_upload_action"] form_path = self.cleaned_data["multimedia_form_path"] media_file.file.seek(0) data = media_file.file.read() media = media_class.get_by_data(data) if cache_handler: cache_handler.sync() cache_handler.data["processed_length"] = 1 cache_handler.save() if media: media.attach_data( data, upload_path=media_file.name, username=username, replace_attachment=replace_attachment ) media.add_domain(domain, owner=True) app.create_mapping(media, form_path) if cache_handler: cache_handler.sync() cache_handler.data["processed"] = 1 cache_handler.save() old_ref = self.cleaned_data["old_reference"] new_ref = reverse("hqmedia_download", args=[domain, media.doc_type, media._id]) return {"successful": {old_ref: new_ref}}
def _get_couchdb_counts(domain): couch_db_counts = Counter() for provider in DOC_PROVIDERS: if isinstance(provider, DocTypeIDProvider): doc_type = provider.doc_type if doc_type == 'CommCareUser': continue # want to split deleted doc_class = get_document_class_by_doc_type(doc_type) count = get_doc_count_in_domain_by_class(domain, doc_class) couch_db_counts.update({doc_type: count}) for _ in CommCareMultimedia.get_db().view('hqmedia/by_domain', key=domain, include_docs=False): couch_db_counts.update(['CommCareMultimedia']) mobile_user_count = get_mobile_user_count(domain) couch_db_counts.update({ 'WebUser': get_web_user_count(domain), 'CommCareUser': mobile_user_count, 'CommCareUser-Deleted': get_doc_count_in_domain_by_class(domain, CommCareUser) - mobile_user_count }) return couch_db_counts
def _fetch_remote_media(local_domain, missing_media, remote_app_details): for filename, item in missing_media: media_class = CommCareMultimedia.get_doc_class(item['media_type']) content = _fetch_remote_media_content(item, remote_app_details) media_item = media_class.get_by_data(content) media_item._id = item['multimedia_id'] media_item.attach_data(content, original_filename=filename) media_item.add_domain(local_domain, owner=True)
def fetch_remote_media(local_domain, missing_media, remote_app_details): for filename, item in missing_media: media_class = CommCareMultimedia.get_doc_class(item['media_type']) content = _fetch_remote_media_content(item, remote_app_details) media_item = media_class.get_by_data(content) media_item._id = item['multimedia_id'] media_item.attach_data(content, original_filename=filename) media_item.add_domain(local_domain, owner=True)
def mime_type(self): try: data = self.uploaded_file.file.read() return CommCareMultimedia.get_mime_type( data, filename=self.uploaded_file.name) except Exception as e: raise BadMediaFileException( "There was an error fetching the MIME type of your file. Error: %s" % e)
def page_context(self): context = super(MultimediaTranslationsCoverageView, self).page_context selected_build_id = self.request.POST.get('build_id') selected_build_text = '' if selected_build_id: build = get_app(self.app.domain, selected_build_id) selected_build_text = str(build.version) if build.build_comment: selected_build_text += ": " + build.build_comment context.update({ "media_types": {t: CommCareMultimedia.get_doc_class(t).get_nice_name() for t in CommCareMultimedia.get_doc_types()}, "selected_langs": self.request.POST.getlist('langs', []), "selected_media_types": self.request.POST.getlist('media_types', ['CommCareAudio', 'CommCareVideo']), "selected_build_id": selected_build_id, "selected_build_text": selected_build_text, }) return context
def _get_missing_multimedia(app): missing = [] for path, media_info in app.multimedia_map.items(): try: local_media = CommCareMultimedia.get(media_info['multimedia_id']) except ResourceNotFound: filename = path.split('/')[-1] missing.append((filename, media_info)) else: _check_domain_access(app.domain, local_media) return missing
def clean_media_file(self): if "media_file" in self.cleaned_data: media_file = self.cleaned_data["media_file"] mime = magic.Magic(mime=True) data = media_file.file.read() content_type = mime.from_buffer(data) if content_type == "application/zip": raise forms.ValidationError("Please use the zip file uploader to upload zip files.") media = CommCareMultimedia.get_doc_class(self.cleaned_data["multimedia_class"]) if not media.validate_content_type(content_type): raise forms.ValidationError("That was not a valid file type, please try again with a different file.") return media_file
def _get_missing_multimedia(app, old_multimedia_ids=None): missing = [] for path, media_info in app.multimedia_map.items(): if old_multimedia_ids and media_info['multimedia_id'] in old_multimedia_ids: continue try: local_media = CommCareMultimedia.get(media_info['multimedia_id']) except ResourceNotFound: filename = path.split('/')[-1] missing.append((filename, media_info)) else: _add_domain_access(app.domain, local_media) return missing
def template_width(self): """ Set column width to accommodate widest image. """ width = 0 if self.app.enable_case_list_icon_dynamic_width: for i, item in enumerate(self.column.enum): for path in item.value.values(): map_item = self.app.multimedia_map[path] if map_item is not None: image = CommCareMultimedia.get(map_item.multimedia_id) if image is not None: for media in image.aux_media: width = max(width, media.media_meta["size"]["width"]) if width == 0: return "13%" return str(width)
def template_width(self): ''' Set column width to accommodate widest image. ''' width = 0 if self.app.enable_case_list_icon_dynamic_width: for i, item in enumerate(self.column.enum): for path in item.value.values(): map_item = self.app.multimedia_map[path] if map_item is not None: image = CommCareMultimedia.get(map_item.multimedia_id) if image is not None: for media in image.aux_media: width = max(width, media.media_meta['size']['width']) if width == 0: return '13%' return str(width)
def test_fetch_missing_media(self): image_path = 'jr://file/commcare/case_list_image.jpg' self.master_app_with_report_modules.get_module(0).set_icon('en', image_path) self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False) remote_details = RemoteLinkDetails( 'http://localhost:8000', 'user', 'key' ) data = b'this is a test: \255' # Real data will be a binary multimedia file, so mock it with bytes, not unicode media_details = list(self.master_app_with_report_modules.multimedia_map.values())[0] media_details['multimedia_id'] = uuid.uuid4().hex media_details['media_type'] = 'CommCareMultimedia' with patch('corehq.apps.linked_domain.remote_accessors._fetch_remote_media_content') as mock: mock.return_value = data _fetch_remote_media('domain', [('case_list_image.jpg', media_details)], remote_details) media = CommCareMultimedia.get(media_details['multimedia_id']) self.addCleanup(media.delete) content = media.fetch_attachment(list(media.blobs.keys())[0]) self.assertEqual(data, content)
def test_fetch_missing_media(self): image_path = 'jr://file/commcare/case_list_image.jpg' self.master_app_with_report_modules.get_module(0).set_icon('en', image_path) self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False) remote_app_details = RemoteAppDetails( 'http://localhost:8000', 'test_domain', 'user', 'key', self.master_app_with_report_modules._id ) data = 'this is a test' media_details = self.master_app_with_report_modules.multimedia_map.values()[0] media_details['multimedia_id'] = uuid.uuid4().hex media_details['media_type'] = 'CommCareMultimedia' with patch('corehq.apps.app_manager.remote_link_accessors._fetch_remote_media_content') as mock: mock.return_value = data _fetch_remote_media('domain', [('case_list_image.jpg', media_details)], remote_app_details) media = CommCareMultimedia.get(media_details['multimedia_id']) self.addCleanup(media.delete) content = media.fetch_attachment(media.blobs.keys()[0]) self.assertEqual(data, content)
def mime_type(self): try: data = self.uploaded_file.file.read() return CommCareMultimedia.get_mime_type(data, filename=self.uploaded_file.name) except Exception as e: raise BadMediaFileException("There was an error fetching the MIME type of your file. Error: %s" % e)
def media_class(self): media_type = self.kwargs.get('media_type') try: return CommCareMultimedia.get_doc_class(media_type) except KeyError: raise Http404("Could not find media of that type.")
def process_bulk_upload_zip(processing_id, domain, app_id, username=None, share_media=False, license_name=None, author=None, attribution_notes=None, replace_existing=False): """ Responsible for processing the uploaded zip from Bulk Upload. """ status = BulkMultimediaStatusCache.get(processing_id) if not status: # no download data available, abort return app = get_app(domain, app_id) status.in_celery = True status.save() try: saved_file = StringIO.StringIO() saved_ref = DownloadBase.get(processing_id) data = saved_ref.get_content() saved_file.write(data) except Exception as e: status.mark_with_error(_("Could not fetch cached bulk upload file. Error: %s." % e)) return try: saved_file.seek(0) uploaded_zip = zipfile.ZipFile(saved_file) except Exception as e: status.mark_with_error(_("Error opening file as zip file: %s" % e)) return if uploaded_zip.testzip(): status.mark_with_error(_("Error encountered processing Zip File. File doesn't look valid.")) return zipped_files = uploaded_zip.namelist() status.total_files = len(zipped_files) checked_paths = [] try: for index, path in enumerate(zipped_files): status.update_progress(len(checked_paths)) checked_paths.append(path) file_name = os.path.basename(path) try: data = uploaded_zip.read(path) except Exception as e: status.add_unmatched_path(path, _("Error reading file: %s" % e)) continue media_class = CommCareMultimedia.get_class_by_data(data) if not media_class: # skip these... continue app_paths = app.get_all_paths_of_type(media_class.__name__) form_path = media_class.get_form_path(path) if not form_path in app_paths: status.add_unmatched_path(path, _("Did not match any %s paths in application." % media_class.get_nice_name())) continue multimedia = media_class.get_by_data(data) if not multimedia: status.add_unmatched_path(path, _("Matching path found, but could not save the data to couch.")) continue is_updated = multimedia.attach_data(data, original_filename=file_name, username=username, replace_attachment=replace_existing) if not is_updated and not getattr(multimedia, '_id'): status.add_unmatched_path(form_path, _("Matching path found, but didn't save new multimedia correctly.")) continue if is_updated: multimedia.add_domain(domain, owner=True) if share_media: multimedia.update_or_add_license(domain, type=license_name, author=author, attribution_notes=attribution_notes) app.create_mapping(multimedia, form_path) media_info = multimedia.get_media_info(form_path, is_updated=is_updated, original_path=path) status.add_matched_path(media_class, media_info) status.update_progress(len(checked_paths)) except Exception as e: status.mark_with_error(_("Error while processing zip: %s" % e)) uploaded_zip.close() status.complete = True status.save()
def process_bulk_upload_zip(processing_id, domain, app_id, username=None, share_media=False, license_name=None, author=None, attribution_notes=None): """ Responsible for processing the uploaded zip from Bulk Upload. """ status = BulkMultimediaStatusCache.get(processing_id) if not status: # no download data available, abort return app = get_app(domain, app_id) status.in_celery = True status.save() uploaded_zip = status.get_upload_zip() if not uploaded_zip: return zipped_files = uploaded_zip.namelist() status.total_files = len(zipped_files) checked_paths = [] try: for index, path in enumerate(zipped_files): status.update_progress(len(checked_paths)) checked_paths.append(path) file_name = os.path.basename(path) try: data = uploaded_zip.read(path) except Exception as e: status.add_unmatched_path(path, _("Error reading file: %s" % e)) continue media_class = CommCareMultimedia.get_class_by_data(data, filename=path) if not media_class: status.add_skipped_path(path, CommCareMultimedia.get_mime_type(data)) continue app_paths = list(app.get_all_paths_of_type(media_class.__name__)) app_paths_lower = [p.lower() for p in app_paths] form_path = media_class.get_form_path(path, lowercase=True) if not form_path in app_paths_lower: status.add_unmatched_path( path, _("Did not match any %s paths in application." % media_class.get_nice_name())) continue index_of_path = app_paths_lower.index(form_path) form_path = app_paths[ index_of_path] # this is the correct capitalization as specified in the form multimedia = media_class.get_by_data(data) if not multimedia: status.add_unmatched_path( path, _("Matching path found, but could not save the data to couch." )) continue is_new = form_path not in list(app.multimedia_map) is_updated = multimedia.attach_data(data, original_filename=file_name, username=username) if not is_updated and not getattr(multimedia, '_id'): status.add_unmatched_path( form_path, _("Matching path found, but didn't save new multimedia correctly." )) continue if is_updated or is_new: multimedia.add_domain(domain, owner=True) if share_media: multimedia.update_or_add_license( domain, type=license_name, author=author, attribution_notes=attribution_notes) app.create_mapping(multimedia, form_path) media_info = multimedia.get_media_info(form_path, is_updated=is_updated, original_path=path) status.add_matched_path(media_class, media_info) status.update_progress(len(checked_paths)) except Exception as e: status.mark_with_error(_("Error while processing zip: %s" % e)) uploaded_zip.close() status.complete = True status.save()
def process_bulk_upload_zip(processing_id, domain, app_id, username=None, share_media=False, license_name=None, author=None, attribution_notes=None): """ Responsible for processing the uploaded zip from Bulk Upload. """ status = BulkMultimediaStatusCache.get(processing_id) if not status: # no download data available, abort return app = get_app(domain, app_id) status.in_celery = True status.save() uploaded_zip = status.get_upload_zip() if not uploaded_zip: return zipped_files = uploaded_zip.namelist() status.total_files = len(zipped_files) checked_paths = [] try: for index, path in enumerate(zipped_files): status.update_progress(len(checked_paths)) checked_paths.append(path) file_name = os.path.basename(path) try: data = uploaded_zip.read(path) except Exception as e: status.add_unmatched_path(path, _("Error reading file: %s" % e)) continue media_class = CommCareMultimedia.get_class_by_data(data, filename=path) if not media_class: status.add_skipped_path(path, CommCareMultimedia.get_mime_type(data)) continue app_paths = list(app.get_all_paths_of_type(media_class.__name__)) app_paths_lower = [p.lower() for p in app_paths] form_path = media_class.get_form_path(path, lowercase=True) if not form_path in app_paths_lower: status.add_unmatched_path(path, _("Did not match any %s paths in application." % media_class.get_nice_name())) continue index_of_path = app_paths_lower.index(form_path) form_path = app_paths[index_of_path] # this is the correct capitalization as specified in the form multimedia = media_class.get_by_data(data) if not multimedia: status.add_unmatched_path(path, _("Matching path found, but could not save the data to couch.")) continue is_new = form_path not in list(app.multimedia_map) is_updated = multimedia.attach_data(data, original_filename=file_name, username=username) if not is_updated and not getattr(multimedia, '_id'): status.add_unmatched_path(form_path, _("Matching path found, but didn't save new multimedia correctly.")) continue if is_updated or is_new: multimedia.add_domain(domain, owner=True) if share_media: multimedia.update_or_add_license(domain, type=license_name, author=author, attribution_notes=attribution_notes) app.create_mapping(multimedia, form_path) media_info = multimedia.get_media_info(form_path, is_updated=is_updated, original_path=path) status.add_matched_path(media_class, media_info) status.update_progress(len(checked_paths)) except Exception as e: status.mark_with_error(_("Error while processing zip: %s" % e)) uploaded_zip.close() status.complete = True status.save()
def all_media(self): from corehq.apps.hqmedia.models import CommCareMultimedia return CommCareMultimedia.view('hqmedia/by_domain', key=self.name, include_docs=True).all()