def process_entity(self, entity):
""" Verifies entity, fetches from journal if necessary and calls
dump_entity.
Args:
entity: The entity to be backed up.
Returns:
True on success, False otherwise.
"""
key = entity.keys()[0]
kind = entity_utils.get_kind_from_entity_key(key)
# Skip protected and private entities.
if re.match(self.PROTECTED_KINDS, kind) or\
re.match(self.PRIVATE_KINDS, kind):
# Do not skip blob entities.
if not re.match(self.BLOB_CHUNK_REGEX, kind) and\
not re.match(self.BLOB_INFO_REGEX, kind):
logging.debug("Skipping key: {0}".format(key))
return False
one_entity = entity[key][dbconstants.APP_ENTITY_SCHEMA[0]]
if one_entity == datastore_server.TOMBSTONE:
return False
app_prefix = entity_utils.get_prefix_from_entity_key(key)
root_key = entity_utils.get_root_key_from_entity_key(key)
success = True
while True:
# Acquire lock.
txn_id = self.zoo_keeper.get_transaction_id(app_prefix)
try:
if self.zoo_keeper.acquire_lock(app_prefix, txn_id, root_key):
version = entity[key][dbconstants.APP_ENTITY_SCHEMA[1]]
if not self.verify_entity(key, version):
# Fetch from the journal.
entity = entity_utils.fetch_journal_entry(self.db_access, key)
if not entity:
logging.error("Bad journal entry for key: {0} and result: {1}".
format(key, entity))
success = False
else:
one_entity = entity[key][dbconstants.APP_ENTITY_SCHEMA[0]]
if self.dump_entity(one_entity):
logging.debug("Backed up key: {0}".format(key))
success = True
else:
success = False
else:
logging.warn("Entity with key: {0} not found".format(key))
success = False
except zk.ZKTransactionException, zk_exception:
logging.error("Zookeeper exception {0} while requesting entity lock".
format(zk_exception))
success = False
except zk.ZKInternalException, zk_exception:
logging.error("Zookeeper exception {0} while requesting entity lock".
format(zk_exception))
success = False
def test_fetch_journal_entry(self):
flexmock(FakeDatastore()).should_receive('batch_get_entity').and_return({})
self.assertEquals(None,
entity_utils.fetch_journal_entry(FakeDatastore(), 'key'))
flexmock(FakeDatastore()).should_receive('batch_get_entity').\
and_return(FAKE_SERIALIZED_ENTITY)
flexmock(entity_pb).should_receive('EntityProto').\
and_return()
def fix_badlisted_entity(self, key, version):
""" Places the correct entity given the current one is from a blacklisted
transaction.
Args:
key: The key to the entity table.
version: The bad version of the entity.
Returns:
True on success, False otherwise.
"""
app_prefix = entity_utils.get_prefix_from_entity_key(key)
root_key = entity_utils.get_root_key_from_entity_key(key)
# TODO watch out for the race condition of doing a GET then a PUT.
try:
txn_id = self.zoo_keeper.get_transaction_id(app_prefix)
if self.zoo_keeper.acquire_lock(app_prefix, txn_id, root_key):
valid_id = self.zoo_keeper.get_valid_transaction_id(app_prefix,
version, key)
# Insert the entity along with regular indexes and composites.
ds_distributed = self.register_db_accessor(app_prefix)
bad_key = datastore_server.DatastoreDistributed.get_journal_key(key,
version)
good_key = datastore_server.DatastoreDistributed.get_journal_key(key,
valid_id)
# Fetch the journal and replace the bad entity.
good_entry = entity_utils.fetch_journal_entry(self.db_access, good_key)
bad_entry = entity_utils.fetch_journal_entry(self.db_access, bad_key)
# Get the kind to lookup composite indexes.
kind = None
if good_entry:
kind = datastore_server.DatastoreDistributed.get_entity_kind(
good_entry.key())
elif bad_entry:
kind = datastore_server.DatastoreDistributed.get_entity_kind(
bad_entry.key())
# Fetch latest composites for this entity
composites = self.get_composite_indexes(app_prefix, kind)
# Remove previous regular indexes and composites if it's not a
# TOMBSTONE.
if bad_entry:
self.delete_indexes(bad_entry)
self.delete_composite_indexes(bad_entry, composites)
# Overwrite the entity table with the correct version.
# Insert into entity table, regular indexes, and composites.
if good_entry:
# TODO
#self.db_access.batch_put_entities(...)
#self.insert_indexes(good_entry)
#self.insert_composite_indexes(good_entry, composites)
pass
else:
# TODO
#self.db_access.batch_delete_entities(...)
pass
del ds_distributed
else:
success = False
except zk.ZKTransactionException, zk_exception:
logging.error("Caught exception {0}".format(zk_exception))
success = False
def fix_badlisted_entity(self, key, version):
""" Places the correct entity given the current one is from a blacklisted
transaction.
Args:
key: The key to the entity table.
version: The bad version of the entity.
Returns:
True on success, False otherwise.
"""
app_prefix = entity_utils.get_prefix_from_entity_key(key)
root_key = entity_utils.get_root_key_from_entity_key(key)
# TODO watch out for the race condition of doing a GET then a PUT.
try:
txn_id = self.zoo_keeper.get_transaction_id(app_prefix)
if self.zoo_keeper.acquire_lock(app_prefix, txn_id, root_key):
valid_id = self.zoo_keeper.get_valid_transaction_id(
app_prefix, version, key)
# Insert the entity along with regular indexes and composites.
ds_distributed = self.register_db_accessor(app_prefix)
bad_key = datastore_server.DatastoreDistributed.get_journal_key(
key, version)
good_key = datastore_server.DatastoreDistributed.get_journal_key(
key, valid_id)
# Fetch the journal and replace the bad entity.
good_entry = entity_utils.fetch_journal_entry(
self.db_access, good_key)
bad_entry = entity_utils.fetch_journal_entry(
self.db_access, bad_key)
# Get the kind to lookup composite indexes.
kind = None
if good_entry:
kind = datastore_server.DatastoreDistributed.get_entity_kind(
good_entry.key())
elif bad_entry:
kind = datastore_server.DatastoreDistributed.get_entity_kind(
bad_entry.key())
# Fetch latest composites for this entity
composites = self.get_composite_indexes(app_prefix, kind)
# Remove previous regular indexes and composites if it's not a
# TOMBSTONE.
if bad_entry:
self.delete_indexes(bad_entry)
self.delete_composite_indexes(bad_entry, composites)
# Overwrite the entity table with the correct version.
# Insert into entity table, regular indexes, and composites.
if good_entry:
# TODO
#self.db_access.batch_put_entities(...)
#self.insert_indexes(good_entry)
#self.insert_composite_indexes(good_entry, composites)
pass
else:
# TODO
#self.db_access.batch_delete_entities(...)
pass
del ds_distributed
else:
success = False
except zk.ZKTransactionException, zk_exception:
logging.error("Caught exception {0}".format(zk_exception))
success = False