def _check_legacy_odk_files(self, build): self.assertTrue(build.copy_of) with self.assertRaises(AttributeError): build.odk_profile_created_after_build path = 'files/profile.ccpr' build_version = build.version build.delete_attachment(path) add_odk_profile_after_build(build) build.save() build = Application.get(build.get_id) self.assertEqual(build.version, build_version) self.assertTrue(build.fetch_attachment(path)) self.assertEqual(build.odk_profile_created_after_build, True)
def download_file(request, domain, app_id, path): download_target_version = request.GET.get( 'download_target_version') == 'true' if download_target_version: parts = path.split('.') assert len(parts) == 2 target = Application.get(app_id).commcare_flavor assert target != 'none' path = parts[0] + '-' + target + '.' + parts[1] if path == "app.json": return JsonResponse(request.app.to_json()) content_type_map = { 'ccpr': 'commcare/profile', 'jad': 'text/vnd.sun.j2me.app-descriptor', 'jar': 'application/java-archive', 'xml': 'application/xml', 'txt': 'text/plain', } try: content_type = content_type_map[path.split('.')[-1]] except KeyError: content_type = None response = HttpResponse(content_type=content_type) if request.GET.get('download') == 'true': response['Content-Disposition'] = "attachment; filename={}".format( path) build_profile = request.GET.get('profile') build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES) if path in ('CommCare.jad', 'CommCare.jar'): set_file_download(response, path) full_path = path elif build_profile and build_profile in request.app.build_profiles and build_profile_access: full_path = 'files/%s/%s' % (build_profile, path) else: full_path = 'files/%s' % path def resolve_path(path): return RegexURLResolver( r'^', 'corehq.apps.app_manager.download_urls').resolve(path) def create_build_files_if_necessary_handling_conflicts(is_retry=False): try: try: # look for file guaranteed to exist if profile is created request.app.fetch_attachment( 'files/{id}/profile.xml'.format(id=build_profile)) except ResourceNotFound: request.app.create_build_files(build_profile_id=build_profile) request.app.save() except ResourceConflict: if is_retry: raise request.app = Application.get(request.app.get_id) create_build_files_if_necessary_handling_conflicts(True) try: assert request.app.copy_of # lazily create language profiles to avoid slowing initial build try: payload = request.app.fetch_attachment(full_path) except ResourceNotFound: if build_profile in request.app.build_profiles and build_profile_access: create_build_files_if_necessary_handling_conflicts() payload = request.app.fetch_attachment(full_path) else: raise if path in ['profile.xml', 'media_profile.xml']: payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint) response.write(payload) response['Content-Length'] = len(response.content) return response except (ResourceNotFound, AssertionError): if request.app.copy_of: if request.META.get('HTTP_USER_AGENT') == 'bitlybot': raise Http404() elif path == 'profile.ccpr': # legacy: should patch build to add odk profile # which wasn't made on build for a long time add_odk_profile_after_build(request.app) request.app.save() return download_file(request, domain, app_id, path) elif path in ('CommCare.jad', 'CommCare.jar'): if not request.app.build_spec.supports_j2me(): raise Http404() request.app.create_jadjar_from_build_files(save=True) try: request.app.save(increment_version=False) except ResourceConflict: # Likely that somebody tried to download the jad and jar # files for the first time simultaneously. pass return download_file(request, domain, app_id, path) else: try: resolve_path(path) except Resolver404: # ok this was just a url that doesn't exist pass else: # this resource should exist but doesn't _assert = soft_assert('@'.join(['jschweers', 'dimagi.com'])) _assert(False, 'Expected build resource %s not found' % path) raise Http404() try: callback, callback_args, callback_kwargs = resolve_path(path) except Resolver404: raise Http404() return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path): if path == "app.json": return JsonResponse(request.app.to_json()) content_type_map = { 'ccpr': 'commcare/profile', 'jad': 'text/vnd.sun.j2me.app-descriptor', 'jar': 'application/java-archive', 'xml': 'application/xml', 'txt': 'text/plain', } try: content_type = content_type_map[path.split('.')[-1]] except KeyError: content_type = None response = HttpResponse(content_type=content_type) build_profile = request.GET.get('profile') build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES) if path in ('CommCare.jad', 'CommCare.jar'): set_file_download(response, path) full_path = path elif build_profile and build_profile in request.app.build_profiles and build_profile_access: full_path = 'files/%s/%s' % (build_profile, path) else: full_path = 'files/%s' % path def resolve_path(path): return RegexURLResolver( r'^', 'corehq.apps.app_manager.download_urls').resolve(path) try: assert request.app.copy_of obj = CachedObject('{id}::{path}'.format( id=request.app._id, path=full_path, )) if not obj.is_cached(): #lazily create language profiles to avoid slowing initial build try: payload = request.app.fetch_attachment(full_path) except ResourceNotFound: if build_profile in request.app.build_profiles and build_profile_access: try: # look for file guaranteed to exist if profile is created request.app.fetch_attachment( 'files/{id}/profile.xml'.format(id=build_profile)) except ResourceNotFound: request.app.create_build_files( save=True, build_profile_id=build_profile) request.app.save() payload = request.app.fetch_attachment(full_path) else: # if profile.xml is found the profile has been built and its a bad request raise else: raise if type(payload) is unicode: payload = payload.encode('utf-8') buffer = StringIO(payload) metadata = {'content_type': content_type} obj.cache_put(buffer, metadata, timeout=None) else: _, buffer = obj.get() payload = buffer.getvalue() if path in ['profile.xml', 'media_profile.xml']: payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint) response.write(payload) response['Content-Length'] = len(response.content) return response except (ResourceNotFound, AssertionError): if request.app.copy_of: if request.META.get('HTTP_USER_AGENT') == 'bitlybot': raise Http404() elif path == 'profile.ccpr': # legacy: should patch build to add odk profile # which wasn't made on build for a long time add_odk_profile_after_build(request.app) request.app.save() return download_file(request, domain, app_id, path) elif path in ('CommCare.jad', 'CommCare.jar'): request.app.create_jadjar_from_build_files(save=True) try: request.app.save(increment_version=False) except ResourceConflict: # Likely that somebody tried to download the jad and jar # files for the first time simultaneously. pass return download_file(request, domain, app_id, path) else: try: resolve_path(path) except Resolver404: # ok this was just a url that doesn't exist # todo: log since it likely exposes a mobile bug # logging was removed because such a mobile bug existed # and was spamming our emails pass else: # this resource should exist but doesn't logging.error('Expected build resource %s not found' % path, extra={'request': request}) if not request.app.build_broken: request.app.build_broken = True request.app.build_broken_reason = 'incomplete-build' try: request.app.save() except ResourceConflict: # this really isn't a big deal: # It'll get updated next time a resource is request'd; # in fact the conflict is almost certainly from # another thread doing this exact update pass raise Http404() try: callback, callback_args, callback_kwargs = resolve_path(path) except Resolver404: raise Http404() return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path): download_target_version = request.GET.get('download_target_version') == 'true' if download_target_version: parts = path.split('.') assert len(parts) == 2 target = Application.get(app_id).commcare_flavor assert target != 'none' path = parts[0] + '-' + target + '.' + parts[1] if path == "app.json": return JsonResponse(request.app.to_json()) content_type_map = { 'ccpr': 'commcare/profile', 'jad': 'text/vnd.sun.j2me.app-descriptor', 'jar': 'application/java-archive', 'xml': 'application/xml', 'txt': 'text/plain', } try: content_type = content_type_map[path.split('.')[-1]] except KeyError: content_type = None response = HttpResponse(content_type=content_type) if request.GET.get('download') == 'true': response['Content-Disposition'] = "attachment; filename={}".format(path) build_profile_id = _get_build_profile_id(request) build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES) if path in ('CommCare.jad', 'CommCare.jar'): set_file_download(response, path) full_path = path elif build_profile_id and build_profile_id in request.app.build_profiles and build_profile_access: full_path = 'files/%s/%s' % (build_profile_id, path) else: full_path = 'files/%s' % path def resolve_path(path): return url(r'^', include('corehq.apps.app_manager.download_urls')).resolve(path) def create_build_files(build_profile_id=None): request.app.create_build_files(build_profile_id=build_profile_id) request.app.save() def create_build_files_if_necessary_handling_conflicts(is_retry=False): try: try: # look for file guaranteed to exist if profile is created request.app.fetch_attachment('files/{id}/profile.xml'.format(id=build_profile_id)) except ResourceNotFound: create_build_files(build_profile_id) except ResourceConflict: if is_retry: raise request.app = Application.get(request.app.get_id) create_build_files_if_necessary_handling_conflicts(True) # Todo; remove after https://dimagi-dev.atlassian.net/browse/ICDS-1483 is fixed extension = path.split(".")[-1] if extension not in content_type_map.keys(): metrics_counter("commcare.invalid_download_requests", tags={"domain": domain, "extension": extension}) try: assert request.app.copy_of # create build files for default profile if they were not created during initial build # or for language profiles for which build files have not been created yet try: payload = request.app.fetch_attachment(full_path) except ResourceNotFound: if not build_profile_id or (build_profile_id in request.app.build_profiles and build_profile_access): create_build_files_if_necessary_handling_conflicts() else: raise payload = request.app.fetch_attachment(full_path) if path in ['profile.xml', 'media_profile.xml']: payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint) response.write(payload) if path in ['profile.ccpr', 'media_profile.ccpr'] and request.app.last_released: last_released = request.app.last_released.replace(microsecond=0) # mobile doesn't want microseconds last_released = ServerTime(last_released).user_time(pytz.UTC).done().isoformat() response['X-CommCareHQ-AppReleasedOn'] = last_released response['Content-Length'] = len(response.content) return response except (ResourceNotFound, AssertionError): if request.app.copy_of: if request.META.get('HTTP_USER_AGENT') == 'bitlybot': raise Http404() elif path == 'profile.ccpr': # legacy: should patch build to add odk profile # which wasn't made on build for a long time add_odk_profile_after_build(request.app) request.app.save() return download_file(request, domain, app_id, path) elif path in ('CommCare.jad', 'CommCare.jar'): if not request.app.build_spec.supports_j2me(): raise Http404() request.app.create_jadjar_from_build_files(save=True) try: request.app.save(increment_version=False) except ResourceConflict: # Likely that somebody tried to download the jad and jar # files for the first time simultaneously. pass return download_file(request, domain, app_id, path) else: try: resolve_path(path) except Resolver404: # ok this was just a url that doesn't exist pass else: # this resource should exist but doesn't _assert = soft_assert('@'.join(['jschweers', 'dimagi.com'])) _assert(False, 'Expected build resource %s not found' % path) raise Http404() try: callback, callback_args, callback_kwargs = resolve_path(path) except Resolver404: raise Http404() return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path): if path == "app.json": return JsonResponse(request.app.to_json()) content_type_map = { 'ccpr': 'commcare/profile', 'jad': 'text/vnd.sun.j2me.app-descriptor', 'jar': 'application/java-archive', 'xml': 'application/xml', 'txt': 'text/plain', } try: content_type = content_type_map[path.split('.')[-1]] except KeyError: content_type = None response = HttpResponse(content_type=content_type) if path in ('CommCare.jad', 'CommCare.jar'): set_file_download(response, path) full_path = path else: full_path = 'files/%s' % path def resolve_path(path): return RegexURLResolver( r'^', 'corehq.apps.app_manager.download_urls').resolve(path) try: assert request.app.copy_of obj = CachedObject('{id}::{path}'.format( id=request.app._id, path=full_path, )) if not obj.is_cached(): payload = request.app.fetch_attachment(full_path) if type(payload) is unicode: payload = payload.encode('utf-8') buffer = StringIO(payload) metadata = {'content_type': content_type} obj.cache_put(buffer, metadata, timeout=None) else: _, buffer = obj.get() payload = buffer.getvalue() response.write(payload) response['Content-Length'] = len(response.content) return response except (ResourceNotFound, AssertionError): if request.app.copy_of: if request.META.get('HTTP_USER_AGENT') == 'bitlybot': raise Http404() elif path == 'profile.ccpr': # legacy: should patch build to add odk profile # which wasn't made on build for a long time add_odk_profile_after_build(request.app) request.app.save() return download_file(request, domain, app_id, path) elif path in ('CommCare.jad', 'CommCare.jar'): request.app.create_jadjar_from_build_files(save=True) try: request.app.save(increment_version=False) except ResourceConflict: # Likely that somebody tried to download the jad and jar # files for the first time simultaneously. pass return download_file(request, domain, app_id, path) else: try: resolve_path(path) except Resolver404: # ok this was just a url that doesn't exist # todo: log since it likely exposes a mobile bug # logging was removed because such a mobile bug existed # and was spamming our emails pass else: # this resource should exist but doesn't logging.error( 'Expected build resource %s not found' % path, extra={'request': request} ) if not request.app.build_broken: request.app.build_broken = True request.app.build_broken_reason = 'incomplete-build' try: request.app.save() except ResourceConflict: # this really isn't a big deal: # It'll get updated next time a resource is request'd; # in fact the conflict is almost certainly from # another thread doing this exact update pass raise Http404() try: callback, callback_args, callback_kwargs = resolve_path(path) except Resolver404: raise Http404() return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path): if path == "app.json": return JsonResponse(request.app.to_json()) content_type_map = { 'ccpr': 'commcare/profile', 'jad': 'text/vnd.sun.j2me.app-descriptor', 'jar': 'application/java-archive', 'xml': 'application/xml', 'txt': 'text/plain', } try: content_type = content_type_map[path.split('.')[-1]] except KeyError: content_type = None response = HttpResponse(content_type=content_type) build_profile = request.GET.get('profile') build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES) if path in ('CommCare.jad', 'CommCare.jar'): set_file_download(response, path) full_path = path elif build_profile and build_profile in request.app.build_profiles and build_profile_access: full_path = 'files/%s/%s' % (build_profile, path) else: full_path = 'files/%s' % path def resolve_path(path): return RegexURLResolver( r'^', 'corehq.apps.app_manager.download_urls').resolve(path) try: assert request.app.copy_of obj = CachedObject('{id}::{path}'.format( id=request.app._id, path=full_path, )) if not obj.is_cached(): #lazily create language profiles to avoid slowing initial build try: payload = request.app.fetch_attachment(full_path) except ResourceNotFound: if build_profile in request.app.build_profiles and build_profile_access: try: # look for file guaranteed to exist if profile is created request.app.fetch_attachment('files/{id}/profile.xml'.format(id=build_profile)) except ResourceNotFound: request.app.create_build_files(save=True, build_profile_id=build_profile) request.app.save() payload = request.app.fetch_attachment(full_path) else: # if profile.xml is found the profile has been built and its a bad request raise else: raise if type(payload) is unicode: payload = payload.encode('utf-8') buffer = StringIO(payload) metadata = {'content_type': content_type} obj.cache_put(buffer, metadata, timeout=None) else: _, buffer = obj.get() payload = buffer.getvalue() if path in ['profile.xml', 'media_profile.xml']: payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint) response.write(payload) response['Content-Length'] = len(response.content) return response except (ResourceNotFound, AssertionError): if request.app.copy_of: if request.META.get('HTTP_USER_AGENT') == 'bitlybot': raise Http404() elif path == 'profile.ccpr': # legacy: should patch build to add odk profile # which wasn't made on build for a long time add_odk_profile_after_build(request.app) request.app.save() return download_file(request, domain, app_id, path) elif path in ('CommCare.jad', 'CommCare.jar'): if not request.app.build_spec.supports_j2me(): raise Http404() request.app.create_jadjar_from_build_files(save=True) try: request.app.save(increment_version=False) except ResourceConflict: # Likely that somebody tried to download the jad and jar # files for the first time simultaneously. pass return download_file(request, domain, app_id, path) else: try: resolve_path(path) except Resolver404: # ok this was just a url that doesn't exist # todo: log since it likely exposes a mobile bug # logging was removed because such a mobile bug existed # and was spamming our emails pass else: # this resource should exist but doesn't _assert = soft_assert('@'.join(['jschweers', 'dimagi.com'])) _assert(False, 'Expected build resource %s not found' % path) raise Http404() try: callback, callback_args, callback_kwargs = resolve_path(path) except Resolver404: raise Http404() return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path): download_target_version = request.GET.get('download_target_version') == 'true' if download_target_version: parts = path.split('.') assert len(parts) == 2 target = Application.get(app_id).target_commcare_flavor assert target != 'none' path = parts[0] + '-' + target + '.' + parts[1] if path == "app.json": return JsonResponse(request.app.to_json()) content_type_map = { 'ccpr': 'commcare/profile', 'jad': 'text/vnd.sun.j2me.app-descriptor', 'jar': 'application/java-archive', 'xml': 'application/xml', 'txt': 'text/plain', } try: content_type = content_type_map[path.split('.')[-1]] except KeyError: content_type = None response = HttpResponse(content_type=content_type) if request.GET.get('download') == 'true': response['Content-Disposition'] = "attachment; filename={}".format(path) build_profile = request.GET.get('profile') build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES) if path in ('CommCare.jad', 'CommCare.jar'): set_file_download(response, path) full_path = path elif build_profile and build_profile in request.app.build_profiles and build_profile_access: full_path = 'files/%s/%s' % (build_profile, path) else: full_path = 'files/%s' % path def resolve_path(path): return RegexURLResolver( r'^', 'corehq.apps.app_manager.download_urls').resolve(path) def create_build_files_if_necessary_handling_conflicts(is_retry=False): try: try: # look for file guaranteed to exist if profile is created request.app.fetch_attachment('files/{id}/profile.xml'.format(id=build_profile), return_bytes=True) except ResourceNotFound: request.app.create_build_files(build_profile_id=build_profile) request.app.save() except ResourceConflict: if is_retry: raise create_build_files_if_necessary_handling_conflicts(True) try: assert request.app.copy_of # lazily create language profiles to avoid slowing initial build try: payload = request.app.fetch_attachment(full_path, return_bytes=True) except ResourceNotFound: if build_profile in request.app.build_profiles and build_profile_access: create_build_files_if_necessary_handling_conflicts() payload = request.app.fetch_attachment(full_path, return_bytes=True) else: raise if path in ['profile.xml', 'media_profile.xml']: payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint) response.write(payload) response['Content-Length'] = len(response.content) return response except (ResourceNotFound, AssertionError): if request.app.copy_of: if request.META.get('HTTP_USER_AGENT') == 'bitlybot': raise Http404() elif path == 'profile.ccpr': # legacy: should patch build to add odk profile # which wasn't made on build for a long time add_odk_profile_after_build(request.app) request.app.save() return download_file(request, domain, app_id, path) elif path in ('CommCare.jad', 'CommCare.jar'): if not request.app.build_spec.supports_j2me(): raise Http404() request.app.create_jadjar_from_build_files(save=True) try: request.app.save(increment_version=False) except ResourceConflict: # Likely that somebody tried to download the jad and jar # files for the first time simultaneously. pass return download_file(request, domain, app_id, path) else: try: resolve_path(path) except Resolver404: # ok this was just a url that doesn't exist pass else: # this resource should exist but doesn't _assert = soft_assert('@'.join(['jschweers', 'dimagi.com'])) _assert(False, 'Expected build resource %s not found' % path) raise Http404() try: callback, callback_args, callback_kwargs = resolve_path(path) except Resolver404: raise Http404() return callback(request, domain, app_id, *callback_args, **callback_kwargs)