def _check_legacy_odk_files(self, build):
self.assertTrue(build.copy_of)
with self.assertRaises(AttributeError):
build.odk_profile_created_after_build
path = 'files/profile.ccpr'
build_version = build.version
build.delete_attachment(path)
add_odk_profile_after_build(build)
build.save()
build = Application.get(build.get_id)
self.assertEqual(build.version, build_version)
self.assertTrue(build.fetch_attachment(path))
self.assertEqual(build.odk_profile_created_after_build, True)
def _check_legacy_odk_files(self, build):
self.assertTrue(build.copy_of)
with self.assertRaises(AttributeError):
build.odk_profile_created_after_build
path = 'files/profile.ccpr'
build_version = build.version
build.delete_attachment(path)
add_odk_profile_after_build(build)
build.save()
build = Application.get(build.get_id)
self.assertEqual(build.version, build_version)
self.assertTrue(build.fetch_attachment(path))
self.assertEqual(build.odk_profile_created_after_build, True)
def download_file(request, domain, app_id, path):
download_target_version = request.GET.get(
'download_target_version') == 'true'
if download_target_version:
parts = path.split('.')
assert len(parts) == 2
target = Application.get(app_id).commcare_flavor
assert target != 'none'
path = parts[0] + '-' + target + '.' + parts[1]
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
if request.GET.get('download') == 'true':
response['Content-Disposition'] = "attachment; filename={}".format(
path)
build_profile = request.GET.get('profile')
build_profile_access = domain_has_privilege(domain,
privileges.BUILD_PROFILES)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
elif build_profile and build_profile in request.app.build_profiles and build_profile_access:
full_path = 'files/%s/%s' % (build_profile, path)
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
def create_build_files_if_necessary_handling_conflicts(is_retry=False):
try:
try:
# look for file guaranteed to exist if profile is created
request.app.fetch_attachment(
'files/{id}/profile.xml'.format(id=build_profile))
except ResourceNotFound:
request.app.create_build_files(build_profile_id=build_profile)
request.app.save()
except ResourceConflict:
if is_retry:
raise
request.app = Application.get(request.app.get_id)
create_build_files_if_necessary_handling_conflicts(True)
try:
assert request.app.copy_of
# lazily create language profiles to avoid slowing initial build
try:
payload = request.app.fetch_attachment(full_path)
except ResourceNotFound:
if build_profile in request.app.build_profiles and build_profile_access:
create_build_files_if_necessary_handling_conflicts()
payload = request.app.fetch_attachment(full_path)
else:
raise
if path in ['profile.xml', 'media_profile.xml']:
payload = convert_XML_To_J2ME(payload, path,
request.app.use_j2me_endpoint)
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
if not request.app.build_spec.supports_j2me():
raise Http404()
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
pass
else:
# this resource should exist but doesn't
_assert = soft_assert('@'.join(['jschweers',
'dimagi.com']))
_assert(False,
'Expected build resource %s not found' % path)
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args,
**callback_kwargs)
def download_file(request, domain, app_id, path):
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
build_profile = request.GET.get('profile')
build_profile_access = domain_has_privilege(domain,
privileges.BUILD_PROFILES)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
elif build_profile and build_profile in request.app.build_profiles and build_profile_access:
full_path = 'files/%s/%s' % (build_profile, path)
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
try:
assert request.app.copy_of
obj = CachedObject('{id}::{path}'.format(
id=request.app._id,
path=full_path,
))
if not obj.is_cached():
#lazily create language profiles to avoid slowing initial build
try:
payload = request.app.fetch_attachment(full_path)
except ResourceNotFound:
if build_profile in request.app.build_profiles and build_profile_access:
try:
# look for file guaranteed to exist if profile is created
request.app.fetch_attachment(
'files/{id}/profile.xml'.format(id=build_profile))
except ResourceNotFound:
request.app.create_build_files(
save=True, build_profile_id=build_profile)
request.app.save()
payload = request.app.fetch_attachment(full_path)
else:
# if profile.xml is found the profile has been built and its a bad request
raise
else:
raise
if type(payload) is unicode:
payload = payload.encode('utf-8')
buffer = StringIO(payload)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=None)
else:
_, buffer = obj.get()
payload = buffer.getvalue()
if path in ['profile.xml', 'media_profile.xml']:
payload = convert_XML_To_J2ME(payload, path,
request.app.use_j2me_endpoint)
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
# todo: log since it likely exposes a mobile bug
# logging was removed because such a mobile bug existed
# and was spamming our emails
pass
else:
# this resource should exist but doesn't
logging.error('Expected build resource %s not found' %
path,
extra={'request': request})
if not request.app.build_broken:
request.app.build_broken = True
request.app.build_broken_reason = 'incomplete-build'
try:
request.app.save()
except ResourceConflict:
# this really isn't a big deal:
# It'll get updated next time a resource is request'd;
# in fact the conflict is almost certainly from
# another thread doing this exact update
pass
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args,
**callback_kwargs)
def download_file(request, domain, app_id, path):
download_target_version = request.GET.get('download_target_version') == 'true'
if download_target_version:
parts = path.split('.')
assert len(parts) == 2
target = Application.get(app_id).commcare_flavor
assert target != 'none'
path = parts[0] + '-' + target + '.' + parts[1]
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
if request.GET.get('download') == 'true':
response['Content-Disposition'] = "attachment; filename={}".format(path)
build_profile_id = _get_build_profile_id(request)
build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
elif build_profile_id and build_profile_id in request.app.build_profiles and build_profile_access:
full_path = 'files/%s/%s' % (build_profile_id, path)
else:
full_path = 'files/%s' % path
def resolve_path(path):
return url(r'^', include('corehq.apps.app_manager.download_urls')).resolve(path)
def create_build_files(build_profile_id=None):
request.app.create_build_files(build_profile_id=build_profile_id)
request.app.save()
def create_build_files_if_necessary_handling_conflicts(is_retry=False):
try:
try:
# look for file guaranteed to exist if profile is created
request.app.fetch_attachment('files/{id}/profile.xml'.format(id=build_profile_id))
except ResourceNotFound:
create_build_files(build_profile_id)
except ResourceConflict:
if is_retry:
raise
request.app = Application.get(request.app.get_id)
create_build_files_if_necessary_handling_conflicts(True)
# Todo; remove after https://dimagi-dev.atlassian.net/browse/ICDS-1483 is fixed
extension = path.split(".")[-1]
if extension not in content_type_map.keys():
metrics_counter("commcare.invalid_download_requests",
tags={"domain": domain, "extension": extension})
try:
assert request.app.copy_of
# create build files for default profile if they were not created during initial build
# or for language profiles for which build files have not been created yet
try:
payload = request.app.fetch_attachment(full_path)
except ResourceNotFound:
if not build_profile_id or (build_profile_id in request.app.build_profiles and build_profile_access):
create_build_files_if_necessary_handling_conflicts()
else:
raise
payload = request.app.fetch_attachment(full_path)
if path in ['profile.xml', 'media_profile.xml']:
payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint)
response.write(payload)
if path in ['profile.ccpr', 'media_profile.ccpr'] and request.app.last_released:
last_released = request.app.last_released.replace(microsecond=0) # mobile doesn't want microseconds
last_released = ServerTime(last_released).user_time(pytz.UTC).done().isoformat()
response['X-CommCareHQ-AppReleasedOn'] = last_released
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
if not request.app.build_spec.supports_j2me():
raise Http404()
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
pass
else:
# this resource should exist but doesn't
_assert = soft_assert('@'.join(['jschweers', 'dimagi.com']))
_assert(False, 'Expected build resource %s not found' % path)
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path):
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
try:
assert request.app.copy_of
obj = CachedObject('{id}::{path}'.format(
id=request.app._id,
path=full_path,
))
if not obj.is_cached():
payload = request.app.fetch_attachment(full_path)
if type(payload) is unicode:
payload = payload.encode('utf-8')
buffer = StringIO(payload)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=None)
else:
_, buffer = obj.get()
payload = buffer.getvalue()
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
# todo: log since it likely exposes a mobile bug
# logging was removed because such a mobile bug existed
# and was spamming our emails
pass
else:
# this resource should exist but doesn't
logging.error(
'Expected build resource %s not found' % path,
extra={'request': request}
)
if not request.app.build_broken:
request.app.build_broken = True
request.app.build_broken_reason = 'incomplete-build'
try:
request.app.save()
except ResourceConflict:
# this really isn't a big deal:
# It'll get updated next time a resource is request'd;
# in fact the conflict is almost certainly from
# another thread doing this exact update
pass
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path):
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
build_profile = request.GET.get('profile')
build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
elif build_profile and build_profile in request.app.build_profiles and build_profile_access:
full_path = 'files/%s/%s' % (build_profile, path)
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
try:
assert request.app.copy_of
obj = CachedObject('{id}::{path}'.format(
id=request.app._id,
path=full_path,
))
if not obj.is_cached():
#lazily create language profiles to avoid slowing initial build
try:
payload = request.app.fetch_attachment(full_path)
except ResourceNotFound:
if build_profile in request.app.build_profiles and build_profile_access:
try:
# look for file guaranteed to exist if profile is created
request.app.fetch_attachment('files/{id}/profile.xml'.format(id=build_profile))
except ResourceNotFound:
request.app.create_build_files(save=True, build_profile_id=build_profile)
request.app.save()
payload = request.app.fetch_attachment(full_path)
else:
# if profile.xml is found the profile has been built and its a bad request
raise
else:
raise
if type(payload) is unicode:
payload = payload.encode('utf-8')
buffer = StringIO(payload)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=None)
else:
_, buffer = obj.get()
payload = buffer.getvalue()
if path in ['profile.xml', 'media_profile.xml']:
payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint)
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
if not request.app.build_spec.supports_j2me():
raise Http404()
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
# todo: log since it likely exposes a mobile bug
# logging was removed because such a mobile bug existed
# and was spamming our emails
pass
else:
# this resource should exist but doesn't
_assert = soft_assert('@'.join(['jschweers', 'dimagi.com']))
_assert(False, 'Expected build resource %s not found' % path)
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path):
download_target_version = request.GET.get('download_target_version') == 'true'
if download_target_version:
parts = path.split('.')
assert len(parts) == 2
target = Application.get(app_id).target_commcare_flavor
assert target != 'none'
path = parts[0] + '-' + target + '.' + parts[1]
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
if request.GET.get('download') == 'true':
response['Content-Disposition'] = "attachment; filename={}".format(path)
build_profile = request.GET.get('profile')
build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
elif build_profile and build_profile in request.app.build_profiles and build_profile_access:
full_path = 'files/%s/%s' % (build_profile, path)
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
def create_build_files_if_necessary_handling_conflicts(is_retry=False):
try:
try:
# look for file guaranteed to exist if profile is created
request.app.fetch_attachment('files/{id}/profile.xml'.format(id=build_profile), return_bytes=True)
except ResourceNotFound:
request.app.create_build_files(build_profile_id=build_profile)
request.app.save()
except ResourceConflict:
if is_retry:
raise
create_build_files_if_necessary_handling_conflicts(True)
try:
assert request.app.copy_of
# lazily create language profiles to avoid slowing initial build
try:
payload = request.app.fetch_attachment(full_path, return_bytes=True)
except ResourceNotFound:
if build_profile in request.app.build_profiles and build_profile_access:
create_build_files_if_necessary_handling_conflicts()
payload = request.app.fetch_attachment(full_path, return_bytes=True)
else:
raise
if path in ['profile.xml', 'media_profile.xml']:
payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint)
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
if not request.app.build_spec.supports_j2me():
raise Http404()
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
pass
else:
# this resource should exist but doesn't
_assert = soft_assert('@'.join(['jschweers', 'dimagi.com']))
_assert(False, 'Expected build resource %s not found' % path)
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args, **callback_kwargs)
