def set(self, values):
"""
Sets a value for each key passed into the constructor.
"""
if not values:
return values
if not self._policy.should_write_cache() or not self.cache:
LOG.warn("memcache.skip.write",
"Per policy, not writing result to the cache",
**self._log_kwds())
return
self._policy.add_cost('w')
cache_mapping = izip(self.cache_entries, values)
with MemcacheChecker(self.cache):
try:
result = self.cache.set_multi(
dict((ce.get_key(), ce._policy.annotate_result(result))
for ce, result in cache_mapping))
if result:
# this only gets logged by python-memcached
# implementation
LOG.error("memcache.set_multi.write",
"Failed to write %s results" % result,
keys=result,
**self._log_kwds())
except pylibmc.WriteError as e:
LOG.error("memcache.error.set_multi",
"memcache set_multi failure",
error=e,
**self._log_kwds())
result = {}
return values
def _safe_entrypoint(f, *args, **kwds):
"""
Decorator that just makes the function avoid throwing exception
"""
try:
return f(*args, **kwds)
except pylibmc.Error as e:
LOG.error("memcache.%s.error" % f.__name__,
"Error in memcache call",
error=e)
def parse(self, mss):
"""
extract data from the image
exif tags from digital cameras
"""
# exif tags from digital cameras?
self.content.fetch_body(mss)
try:
# XXXarielb move to pygmy as soon as pygmy doesn't crash within threads
from PIL import Image
img = Image.open(StringIO(self.content.body))
# believe the image parser over anything in the graph
self.size = img.size
except ImportError, ie:
LOG.error("format.image.no_pil", str(e))
raise
def raw_get(self):
"""
Gets the raw value from the cache
"""
key = self.get_key()
self._policy.add_cost('r')
# no memcache hooked up?
if not self.cache:
return None
with MemcacheChecker(self.cache):
try:
return self.cache.get(key)
except pylibmc.Error as e:
LOG.error("memcache.error.get",
"memcache get failure",
error=e,
**self._log_kwds())
def raw_set(self, full_result):
"""
Sets the raw value directly in the cache
"""
key = self.get_key()
if not self.cache:
return
self._policy.add_cost('w')
expires, long_lived = self._policy.get_expires()
with MemcacheChecker(self.cache):
try:
return self.cache.set(key, full_result, time=expires)
except (pylibmc.WriteError, pylibmc._pylibmc.MemcachedError) as e:
LOG.error("memcache.error.set",
"memcache set failure",
error=e,
**self._log_kwds())
def set(self, result):
"""
A full, synchronous set-with-policy operation.
"""
if not self._policy.should_write_cache():
LOG.warn("memcache.skip.write",
"Per policy, not writing result to the cache",
**self._log_kwds())
return
full_result = self._policy.annotate_result(result)
success = self.raw_set(full_result)
if not success:
LOG.error("memcache.set.write",
"Failed to write %s" % self.get_key(),
key=self.get_key(),
**self._log_kwds(code=success))
# acts as an identity function so it can be used with Twisted
# deferreds and such
return result
class ImageContent(ContentWrapper):
"""
methods for dealing with image content
"""
# ie6 uses some bizarre content_types for PNG and JPEG images
# XXX it would be nice to fix the content_type in the
# /type/content object, but it may already have been uploaded.
# so for now, images uploaded from ie6 will have the "wrong"
# content-type and we'll need to garden them.
remap_dumb_ie_mime_types = {
'image/pjpeg': contenttype.MediaType('image/jpeg'),
'image/x-png': contenttype.MediaType('image/png')
}
@classmethod
def match(cls, c):
"""
true if this ContentWrapper subclass applies to the content argument.
"""
media_type = cls.remap_dumb_ie_mime_types.get(c.media_type, c.media_type)
if not c.media_type.startswith('image/'):
return False
subtype = media_type.split('/')[1]
return subtype in ('gif', 'png', 'jpeg', 'x-icon')
def __init__(self, content):
super(ImageContent, self).__init__(content)
self.size = None
def load(self, mss):
result = mss.mqlread(dict(id=self.content.content_id,
type='/common/image',
size=dict(x=None, y=None)))
if result is None:
return
self.size = (result['size']['x'], result['size']['y'])
def upload(self, mss):
"""
add a /common/image facet to the type/content
"""
self.load(mss)
if self.size is None:
self.parse(mss)
w = { 'id': self.content.content_id,
'type': { 'connect': 'insert',
'id': '/common/image' }}
if self.size[0] and self.size[1]:
w['/common/image/size'] = { 'create': 'unless_exists',
'type': '/measurement_unit/rect_size',
'x': self.size[0],
'y': self.size[1] }
with mss.push_variables(authority="/user/content_administrator",
privileged=scope.Authority):
result = mss.mqlwrite(w)
def parse(self, mss):
"""
extract data from the image
exif tags from digital cameras
"""
# exif tags from digital cameras?
self.content.fetch_body(mss)
try:
# XXXarielb move to pygmy as soon as pygmy doesn't crash within threads
from PIL import Image
img = Image.open(StringIO(self.content.body))
# believe the image parser over anything in the graph
self.size = img.size
except ImportError, ie:
LOG.error("format.image.no_pil", str(e))
raise
except Exception, e:
LOG.error("format.image.parse", str(e))
raise ContentLoadError('Invalid image file',
app_code="upload/invalid_image_data",
error=e)
media_type = self.content.media_type
LOG.info('update_content', "Image Updating content from %s to %s" % (media_type,
self.remap_dumb_ie_mime_types.get(media_type)))
self.content.media_type = self.remap_dumb_ie_mime_types.get(media_type, media_type)
@classmethod
def get_fallback_image_path(cls):
try:
config = mw.siteconfig.get_config2()
path = config.get('me.img_thumb_fallback')
if path and os.path.exists(path):
return path
except KeyError, e:
pass
LOG.error("image.thumb", "Could not find fallback image for thumbnailing service.")
return None
# failover for thumnailing operation in the event that
# the image is too large to thumbnail
def thumb_fallback(self, mss):
path = ImageContent.get_fallback_image_path()
if path is None:
return None
# load data
fd = open(path)
data = fd.read()
fd.close()
# the fallback image is a known GIF image.
thumb_mt = 'image/gif'
def get(self):
"""
Gets all the cache entries - will return a triple of::
('hit', 'miss' or 'skip', value, CacheEntry)
for each cache entry passed in to the CacheEntryList constructor
"""
# no memcache hooked up?
if not self._policy.should_read_cache() or not self.cache:
LOG.warn("memcache.skip.read", "Skipping cache",
**self._log_kwds())
self._policy.add_cost('s')
return [('skip', None, ce) for ce in self.cache_entries]
self._policy.add_cost('r')
with MemcacheChecker(self.cache):
try:
memcache_result = self.cache.get_multi(
[ce.get_key() for ce in self.cache_entries])
except pylibmc.Error as e:
memcache_result = {}
LOG.error("memcache.error.get_multi",
"memcache get_multi failure",
error=e,
**self._log_kwds())
assert isinstance(memcache_result, dict)
result = []
# create an entry in the result for each cache entry
for ce in self.cache_entries:
key = ce.get_key()
mr = memcache_result.get(key)
if not ce.check_result(key, mr):
result.append(('miss', None, ce))
else:
result.append(('hit', ce._policy.extract_result(mr), ce))
misses = [miss for miss in result if miss[0] == 'miss']
hits = [hit for hit in result if hit[0] == 'hit']
miss_keys = [ce.key_obj for (status, value, ce) in misses]
miss_hashes = [ce.get_key() for (status, value, ce) in misses]
hit_keys = [ce.key_obj for (status, value, ce) in hits]
hit_hashes = [ce.get_key() for (status, value, ce) in hits]
if miss_keys:
code = "hits+misses" if hit_keys else "all miss"
else:
code = "all hit" if hit_keys else "empty"
LOG.notice("%s.cache.multiresult" % self._policy.tag,
"",
miss_hashes=miss_hashes,
hit_hashes=hit_hashes,
miss_count=len(misses),
hit_count=len(hits),
**self._log_kwds(code=code))
return result