def non_transactional_put(self,
filename,
stream,
is_draft=False,
metadata_only=False):
"""Non-transactional put; use only when transactions are impossible."""
filename = self._logical_to_physical(filename)
metadata = FileMetadataEntity.get_by_key_name(filename)
if not metadata:
metadata = FileMetadataEntity(key_name=filename)
metadata.updated_on = datetime.datetime.now()
metadata.is_draft = is_draft
if not metadata_only:
# We operate with raw bytes. The consumer must deal with encoding.
raw_bytes = stream.read()
metadata.size = len(raw_bytes)
data = FileDataEntity(key_name=filename)
data.data = raw_bytes
data.put()
metadata.put()
MemcacheManager.delete(self.make_key(filename), namespace=self._ns)
def _save_to_memcache(self):
"""Saves course representation into memcache."""
envelope = SerializableCourseEnvelope()
envelope.units = self._units
envelope.lessons = self._lessons
envelope.references = self._references
MemcacheManager.set(self.memcache_key, envelope)
def wait_and_finalize():
data_future.check_success()
metadata_future.check_success()
MemcacheManager.delete_multi(
[self.make_key(filename) for filename in filename_list],
namespace=self._ns)
def isfile(self, afilename):
"""Checks file existence by looking up the datastore row."""
filename = self._logical_to_physical(afilename)
# Check cache.
result = MemcacheManager.get(
self.make_key(filename), namespace=self._ns)
if result:
return True
if NO_OBJECT == result:
return False
# Check datastore.
metadata = FileMetadataEntity.get_by_key_name(filename)
if metadata:
return True
result = False
# Check with parent fs.
if self._inherits_from and self._can_inherit(filename):
result = self._inherits_from.isfile(afilename)
# Put NO_OBJECT marker into memcache to avoid repeated lookups.
if not result:
MemcacheManager.set(
self.make_key(filename), NO_OBJECT, namespace=self._ns)
return result
def _save_to_memcache(self):
"""Saves course representation into memcache."""
envelope = SerializableCourseEnvelope()
envelope.units = self._units
envelope.lessons = self._lessons
envelope.references = self._references
MemcacheManager.set(self.memcache_key, envelope)
def isfile(self, afilename):
"""Checks file existence by looking up the datastore row."""
filename = self._logical_to_physical(afilename)
# Check cache.
result = MemcacheManager.get(self.make_key(filename),
namespace=self._ns)
if result:
return True
if NO_OBJECT == result:
return False
# Check datastore.
metadata = FileMetadataEntity.get_by_key_name(filename)
if metadata:
return True
result = False
# Check with parent fs.
if self._inherits_from and self._can_inherit(filename):
result = self._inherits_from.isfile(afilename)
# Put NO_OBJECT marker into memcache to avoid repeated lookups.
if not result:
MemcacheManager.set(self.make_key(filename),
NO_OBJECT,
namespace=self._ns)
return result
def delete(self, filename):
filename = self._logical_to_physical(filename)
metadata = FileMetadataEntity.get_by_key_name(filename)
if metadata:
metadata.delete()
data = FileDataEntity(key_name=filename)
if data:
data.delete()
MemcacheManager.delete(self.make_key(filename), namespace=self._ns)
def delete(self, filename):
filename = self._logical_to_physical(filename)
metadata = FileMetadataEntity.get_by_key_name(filename)
if metadata:
metadata.delete()
data = FileDataEntity(key_name=filename)
if data:
data.delete()
MemcacheManager.delete(self.make_key(filename), namespace=self._ns)
def update_permissions_map(cls):
"""Puts a dictionary mapping users to permissions in memcache.
A dictionary is constructed, using roles information from the datastore,
mapping user emails to dictionaries that map module names to
sets of permissions.
Returns:
The created dictionary.
"""
permissions_map = {}
for role in RoleDAO.get_all():
for user in role.users:
user_permissions = permissions_map.setdefault(user, {})
for (module_name, permissions) in role.permissions.iteritems():
module_permissions = user_permissions.setdefault(module_name, set())
module_permissions.update(permissions)
MemcacheManager.set(cls.memcache_key, permissions_map)
return permissions_map
def update_permissions_map(cls):
"""Puts a dictionary mapping users to permissions in memcache.
A dictionary is constructed, using roles information from the datastore,
mapping user emails to dictionaries that map module names to
sets of permissions.
Returns:
The created dictionary.
"""
permissions_map = {}
for role in RoleDAO.get_all():
for user in role.users:
user_permissions = permissions_map.setdefault(user, {})
for (module_name, permissions) in role.permissions.iteritems():
module_permissions = user_permissions.setdefault(
module_name, set())
module_permissions.update(permissions)
MemcacheManager.set(cls.memcache_key, permissions_map)
return permissions_map
def _load_from_memcache(self):
"""Loads course representation from memcache."""
try:
envelope = MemcacheManager.get(self.memcache_key)
if envelope:
self._units = envelope.units
self._lessons = envelope.lessons
self._reindex()
self._loaded = True
except Exception as e: # pylint: disable-msg=broad-except
logging.error(
'Failed to load course \'%s\' from memcache. %s',
self.memcache_key, e)
def _load_from_memcache(self):
"""Loads course representation from memcache."""
try:
envelope = MemcacheManager.get(self.memcache_key)
if envelope:
self._units = envelope.units
self._lessons = envelope.lessons
self._references = envelope.references
self._reindex()
self._loaded = True
except Exception as e: # pylint: disable-msg=broad-except
logging.error('Failed to load course \'%s\' from memcache. %s',
self.memcache_key, e)
def non_transactional_put(
self, filename, stream, is_draft=False, metadata_only=False):
"""Non-transactional put; use only when transactions are impossible."""
filename = self._logical_to_physical(filename)
metadata = FileMetadataEntity.get_by_key_name(filename)
if not metadata:
metadata = FileMetadataEntity(key_name=filename)
metadata.updated_on = datetime.datetime.now()
metadata.is_draft = is_draft
if not metadata_only:
# We operate with raw bytes. The consumer must deal with encoding.
raw_bytes = stream.read()
metadata.size = len(raw_bytes)
data = FileDataEntity(key_name=filename)
data.data = raw_bytes
data.put()
metadata.put()
MemcacheManager.delete(self.make_key(filename), namespace=self._ns)
def load(cls, app_context):
"""Loads instance from memcache; does not fail on errors."""
try:
binary_data = MemcacheManager.get(
cls._make_key(),
namespace=app_context.get_namespace_name())
if binary_data:
memento = cls.new_memento()
memento.deserialize(binary_data)
return cls.instance_from_memento(app_context, memento)
except Exception as e: # pylint: disable-msg=broad-except
logging.error(
'Failed to load object \'%s\' from memcache. %s',
cls._make_key(), e)
return None
def load(cls, app_context):
"""Loads instance from memcache; does not fail on errors."""
try:
binary_data = MemcacheManager.get(
cls._make_key(),
namespace=app_context.get_namespace_name())
if binary_data:
memento = cls.new_memento()
memento.deserialize(binary_data)
return cls.instance_from_memento(app_context, memento)
except Exception as e: # pylint: disable-msg=broad-except
logging.error(
'Failed to load object \'%s\' from memcache. %s',
cls._make_key(), e)
return None
def get(self, afilename):
"""Gets a file from a datastore. Raw bytes stream, no encodings."""
filename = self._logical_to_physical(afilename)
# Load from cache.
result = MemcacheManager.get(self.make_key(filename),
namespace=self._ns)
if result:
return result
if NO_OBJECT == result:
return None
# Load from a datastore.
metadata = FileMetadataEntity.get_by_key_name(filename)
if metadata:
data = FileDataEntity.get_by_key_name(filename)
if data:
result = FileStreamWrapped(metadata, data.data)
MemcacheManager.set(self.make_key(filename),
result,
namespace=self._ns)
return result
result = None
metadata = None
# Load from parent fs.
if self._inherits_from and self._can_inherit(filename):
result = self._inherits_from.get(afilename)
# Cache result.
if result:
result = FileStreamWrapped(metadata, result.read())
MemcacheManager.set(self.make_key(filename),
result,
namespace=self._ns)
else:
MemcacheManager.set(self.make_key(filename),
NO_OBJECT,
namespace=self._ns)
return result
def get(self, afilename):
"""Gets a file from a datastore. Raw bytes stream, no encodings."""
filename = self._logical_to_physical(afilename)
# Load from cache.
result = MemcacheManager.get(
self.make_key(filename), namespace=self._ns)
if result:
return result
if NO_OBJECT == result:
return None
# Load from a datastore.
metadata = FileMetadataEntity.get_by_key_name(filename)
if metadata:
data = FileDataEntity.get_by_key_name(filename)
if data:
result = FileStreamWrapped(metadata, data.data)
MemcacheManager.set(
self.make_key(filename), result, namespace=self._ns)
return result
result = None
metadata = None
# Load from parent fs.
if self._inherits_from and self._can_inherit(filename):
result = self._inherits_from.get(afilename)
# Cache result.
if result:
result = FileStreamWrapped(metadata, result.read())
MemcacheManager.set(
self.make_key(filename), result, namespace=self._ns)
else:
MemcacheManager.set(
self.make_key(filename), NO_OBJECT, namespace=self._ns)
return result
def _load_permissions_map(cls):
"""Loads the permissions map from Memcache or creates it if needed."""
permissions_map = MemcacheManager.get(cls.memcache_key)
if permissions_map is None: # As opposed to {}, which is valid.
permissions_map = cls.update_permissions_map()
return permissions_map
def save(cls, app_context, instance):
"""Saves instance to memcache."""
MemcacheManager.set(
cls._make_key(),
cls.memento_from_instance(instance).serialize(),
namespace=app_context.get_namespace_name())
def _load_permissions_map(cls):
"""Loads the permissions map from Memcache or creates it if needed."""
permissions_map = MemcacheManager.get(cls.memcache_key)
if not permissions_map:
permissions_map = cls.update_permissions_map()
return permissions_map
def delete(cls, app_context):
"""Deletes instance from memcache."""
MemcacheManager.delete(
cls._make_key(),
namespace=app_context.get_namespace_name())
def save(cls, app_context, instance):
"""Saves instance to memcache."""
MemcacheManager.set(
cls._make_key(),
cls.memento_from_instance(instance).serialize(),
namespace=app_context.get_namespace_name())
def delete(cls, app_context):
"""Deletes instance from memcache."""
MemcacheManager.delete(
cls._make_key(),
namespace=app_context.get_namespace_name())
def _load_permissions_map(cls):
"""Loads the permissions map from Memcache or creates it if needed."""
permissions_map = MemcacheManager.get(cls.memcache_key)
if permissions_map is None: # As opposed to {}, which is valid.
permissions_map = cls.update_permissions_map()
return permissions_map