def get(self, *args, **kwargs):
filename = str(self.request.GET.get('file'))
if not filename in VALID_FILENAMES:
return HttpResponseBadRequest('invalid filename=' + filename)
obj = CachedObject(filename)
if not obj.is_cached():
f = open('corehq/apps/api/data/' + filename, 'r')
text = f.read()
f.close()
buffer = StringIO.StringIO(text)
metadata = {'content_type': 'text/plain'}
obj.cache_put(buffer, metadata)
else:
#raise ValueError("is cached")
pass
cmeta, cstream = obj.get()
#####################
wrapper = FileWrapper(cstream)
if cmeta is not None:
mime_type = cmeta['content_type']
else:
mime_type = "plain/text"
response = HttpResponse(wrapper, mimetype=mime_type)
return response
def test_is_cached_lost_stream_key(self):
text = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
filename = "something"
buffer = StringIO.StringIO(text)
metadata = {'content_type': 'text/plain'}
obj = CachedObject(filename)
obj.cache_put(buffer, metadata)
obj.rcache.delete(obj.stream_key(OBJECT_ORIGINAL))
self.assertFalse(obj.is_cached())
def get(self, request, *args, **kwargs):
obj = CachedObject(str(self.doc_id) + ':' + self.kwargs.get('media_type'))
if not obj.is_cached():
data, content_type = self.multimedia.get_display_file()
if self.thumb:
data = CommCareImage.get_thumbnail_data(data, self.thumb)
buffer = StringIO(data)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=0)
else:
metadata, buffer = obj.get()
data = buffer.getvalue()
content_type = metadata['content_type']
return HttpResponse(data, mimetype=content_type)
def fetch_case_attachment(cls,
case_id,
attachment_key,
fixed_size=None,
**kwargs):
"""
Return (metadata, stream) information of best matching image attachment.
TODO: This should be the primary case_attachment retrieval method, the image one is a silly separation of similar functionality
Additional functionality to be abstracted by content_type of underlying attachment
"""
size_key = OBJECT_ORIGINAL
if fixed_size is not None and fixed_size in OBJECT_SIZE_MAP:
size_key = fixed_size
# if size key is None, then one of the limit criteria are set
attachment_cache_key = "%(case_id)s_%(attachment)s" % {
"case_id": case_id,
"attachment": attachment_key
}
cobject = CachedObject(attachment_cache_key)
meta, stream = cls.cache_and_get_object(cobject,
case_id,
attachment_key,
size_key=size_key)
return meta, stream
def get(self, *args, **kwargs):
filename = str(self.request.GET.get('file'))
if not filename in VALID_FILENAMES:
return HttpResponseBadRequest('invalid filename=' + filename)
obj = CachedObject(filename)
if not obj.is_cached():
f = open('corehq/apps/api/data/' + filename, 'r')
text = f.read()
f.close()
buffer = StringIO.StringIO(text)
metadata = {'content_type': 'text/plain'}
obj.cache_put(buffer, metadata)
else:
#raise ValueError("is cached")
pass
cmeta, cstream = obj.get()
#####################
wrapper = FileWrapper(cstream)
if cmeta is not None:
mime_type = cmeta['content_type']
else:
mime_type = "plain/text"
response = HttpResponse(wrapper, content_type=mime_type)
return response
def get_cached_case_attachment(domain, case_id, attachment_id, is_image=False):
attachment_cache_key = "%(case_id)s_%(attachment)s" % {
"case_id": case_id,
"attachment": attachment_id
}
from dimagi.utils.django.cached_object import CachedObject, CachedImage
cobject = CachedImage(attachment_cache_key) if is_image else CachedObject(attachment_cache_key)
if not cobject.is_cached():
content = CaseAccessors(domain).get_attachment_content(case_id, attachment_id)
stream = StringIO(content.content_body)
metadata = {'content_type': content.content_type}
cobject.cache_put(stream, metadata)
return cobject
def test_is_cached_lost_stream_key(self):
text = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
filename = "something"
buffer = StringIO.StringIO(text)
metadata = {'content_type': 'text/plain'}
obj = CachedObject(filename)
obj.cache_put(buffer, metadata)
obj.rcache.delete(obj.stream_key(OBJECT_ORIGINAL))
self.assertFalse(obj.is_cached())
def testBasicObjects(self):
text = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
filename = "something"
buffer = StringIO.StringIO(text)
obj = CachedObject(filename)
self.assertFalse(obj.is_cached())
metadata = {'content_type': 'text/plain'}
obj.cache_put(buffer, metadata)
self.assertTrue(obj.is_cached())
cmeta, cstream = obj.get()
self.assertEqual(cmeta['content_length'], len(text))
self.assertEqual(cstream.read(), text)
def testBasicObjects(self):
text = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
filename = "something"
buffer = StringIO.StringIO(text)
obj = CachedObject(filename)
self.assertFalse(obj.is_cached())
metadata = {'content_type': 'text/plain'}
obj.cache_put(buffer, metadata)
self.assertTrue(obj.is_cached())
cmeta, cstream = obj.get()
self.assertEqual(cmeta['content_length'], len(text))
self.assertEqual(cstream.read(), text)
def get(self, request, *args, **kwargs):
obj = CachedObject(
str(self.doc_id) + ':' + self.kwargs.get('media_type') + ':' +
str(self.thumb))
if not obj.is_cached():
data, content_type = self.multimedia.get_display_file()
if self.thumb:
data = CommCareImage.get_thumbnail_data(data, self.thumb)
buffer = StringIO(data)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=None)
else:
metadata, buffer = obj.get()
data = buffer.getvalue()
content_type = metadata['content_type']
return HttpResponse(data, content_type=content_type)
def download_file(request, domain, app_id, path):
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
build_profile = request.GET.get('profile')
build_profile_access = domain_has_privilege(domain,
privileges.BUILD_PROFILES)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
elif build_profile and build_profile in request.app.build_profiles and build_profile_access:
full_path = 'files/%s/%s' % (build_profile, path)
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
try:
assert request.app.copy_of
obj = CachedObject('{id}::{path}'.format(
id=request.app._id,
path=full_path,
))
if not obj.is_cached():
#lazily create language profiles to avoid slowing initial build
try:
payload = request.app.fetch_attachment(full_path)
except ResourceNotFound:
if build_profile in request.app.build_profiles and build_profile_access:
try:
# look for file guaranteed to exist if profile is created
request.app.fetch_attachment(
'files/{id}/profile.xml'.format(id=build_profile))
except ResourceNotFound:
request.app.create_build_files(
save=True, build_profile_id=build_profile)
request.app.save()
payload = request.app.fetch_attachment(full_path)
else:
# if profile.xml is found the profile has been built and its a bad request
raise
else:
raise
if type(payload) is unicode:
payload = payload.encode('utf-8')
buffer = StringIO(payload)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=None)
else:
_, buffer = obj.get()
payload = buffer.getvalue()
if path in ['profile.xml', 'media_profile.xml']:
payload = convert_XML_To_J2ME(payload, path,
request.app.use_j2me_endpoint)
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
# todo: log since it likely exposes a mobile bug
# logging was removed because such a mobile bug existed
# and was spamming our emails
pass
else:
# this resource should exist but doesn't
logging.error('Expected build resource %s not found' %
path,
extra={'request': request})
if not request.app.build_broken:
request.app.build_broken = True
request.app.build_broken_reason = 'incomplete-build'
try:
request.app.save()
except ResourceConflict:
# this really isn't a big deal:
# It'll get updated next time a resource is request'd;
# in fact the conflict is almost certainly from
# another thread doing this exact update
pass
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args,
**callback_kwargs)
def download_file(request, domain, app_id, path):
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
try:
assert request.app.copy_of
obj = CachedObject('{id}::{path}'.format(
id=request.app._id,
path=full_path,
))
if not obj.is_cached():
payload = request.app.fetch_attachment(full_path)
if type(payload) is unicode:
payload = payload.encode('utf-8')
buffer = StringIO(payload)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=None)
else:
_, buffer = obj.get()
payload = buffer.getvalue()
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
# todo: log since it likely exposes a mobile bug
# logging was removed because such a mobile bug existed
# and was spamming our emails
pass
else:
# this resource should exist but doesn't
logging.error(
'Expected build resource %s not found' % path,
extra={'request': request}
)
if not request.app.build_broken:
request.app.build_broken = True
request.app.build_broken_reason = 'incomplete-build'
try:
request.app.save()
except ResourceConflict:
# this really isn't a big deal:
# It'll get updated next time a resource is request'd;
# in fact the conflict is almost certainly from
# another thread doing this exact update
pass
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args, **callback_kwargs)
def download_file(request, domain, app_id, path):
if path == "app.json":
return JsonResponse(request.app.to_json())
content_type_map = {
'ccpr': 'commcare/profile',
'jad': 'text/vnd.sun.j2me.app-descriptor',
'jar': 'application/java-archive',
'xml': 'application/xml',
'txt': 'text/plain',
}
try:
content_type = content_type_map[path.split('.')[-1]]
except KeyError:
content_type = None
response = HttpResponse(content_type=content_type)
build_profile = request.GET.get('profile')
build_profile_access = domain_has_privilege(domain, privileges.BUILD_PROFILES)
if path in ('CommCare.jad', 'CommCare.jar'):
set_file_download(response, path)
full_path = path
elif build_profile and build_profile in request.app.build_profiles and build_profile_access:
full_path = 'files/%s/%s' % (build_profile, path)
else:
full_path = 'files/%s' % path
def resolve_path(path):
return RegexURLResolver(
r'^', 'corehq.apps.app_manager.download_urls').resolve(path)
try:
assert request.app.copy_of
obj = CachedObject('{id}::{path}'.format(
id=request.app._id,
path=full_path,
))
if not obj.is_cached():
#lazily create language profiles to avoid slowing initial build
try:
payload = request.app.fetch_attachment(full_path)
except ResourceNotFound:
if build_profile in request.app.build_profiles and build_profile_access:
try:
# look for file guaranteed to exist if profile is created
request.app.fetch_attachment('files/{id}/profile.xml'.format(id=build_profile))
except ResourceNotFound:
request.app.create_build_files(save=True, build_profile_id=build_profile)
request.app.save()
payload = request.app.fetch_attachment(full_path)
else:
# if profile.xml is found the profile has been built and its a bad request
raise
else:
raise
if type(payload) is unicode:
payload = payload.encode('utf-8')
buffer = StringIO(payload)
metadata = {'content_type': content_type}
obj.cache_put(buffer, metadata, timeout=None)
else:
_, buffer = obj.get()
payload = buffer.getvalue()
if path in ['profile.xml', 'media_profile.xml']:
payload = convert_XML_To_J2ME(payload, path, request.app.use_j2me_endpoint)
response.write(payload)
response['Content-Length'] = len(response.content)
return response
except (ResourceNotFound, AssertionError):
if request.app.copy_of:
if request.META.get('HTTP_USER_AGENT') == 'bitlybot':
raise Http404()
elif path == 'profile.ccpr':
# legacy: should patch build to add odk profile
# which wasn't made on build for a long time
add_odk_profile_after_build(request.app)
request.app.save()
return download_file(request, domain, app_id, path)
elif path in ('CommCare.jad', 'CommCare.jar'):
if not request.app.build_spec.supports_j2me():
raise Http404()
request.app.create_jadjar_from_build_files(save=True)
try:
request.app.save(increment_version=False)
except ResourceConflict:
# Likely that somebody tried to download the jad and jar
# files for the first time simultaneously.
pass
return download_file(request, domain, app_id, path)
else:
try:
resolve_path(path)
except Resolver404:
# ok this was just a url that doesn't exist
# todo: log since it likely exposes a mobile bug
# logging was removed because such a mobile bug existed
# and was spamming our emails
pass
else:
# this resource should exist but doesn't
_assert = soft_assert('@'.join(['jschweers', 'dimagi.com']))
_assert(False, 'Expected build resource %s not found' % path)
raise Http404()
try:
callback, callback_args, callback_kwargs = resolve_path(path)
except Resolver404:
raise Http404()
return callback(request, domain, app_id, *callback_args, **callback_kwargs)
