def test_clean_removes_markers_with_different_values_on_non_default_namespace(self):
self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1)
self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2)
NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"]
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i3.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=NS1_NAMESPACE)
default_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
original_marker = datastore.Get(marker_key)
default_marker = datastore.Get(default_key)
marker2 = "{}|name:{}".format(TestModel._meta.db_table, md5("bananas").hexdigest())
new_marker = datastore.Entity(UniqueMarker.kind(), name=marker2, namespace=NS1_NAMESPACE)
new_marker.update(original_marker)
datastore.Put(new_marker)
# This allows us to test: 1) namespaced markers will check against their namespace models (not all of them)"
self.i1.delete()
#... 2) the mapper only cleans the desired namespace
datastore.Put(default_marker)
UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel), db="ns1")
process_task_queues()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, new_marker.key())
self.assertTrue(datastore.Get(default_marker.key()))
self.assertTrue(datastore.Get(marker_key))
datastore.Delete(default_marker)
def test_constraints_can_be_enabled_per_model(self):
initial_count = datastore.Query(UniqueMarker.kind()).Count()
instance1 = ModelWithUniquesAndOverride.objects.create(name="One")
self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
def test_delete_clears_markers(self):
initial_count = datastore.Query(UniqueMarker.kind()).Count()
instance = ModelWithUniques.objects.create(name="One")
self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
instance.delete()
self.assertEqual(0, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
def test_update_updates_markers(self):
initial_count = datastore.Query(UniqueMarker.kind()).Count()
instance = ModelWithUniques.objects.create(name="One")
self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
qry = datastore.Query(UniqueMarker.kind())
qry.Order(("created", datastore.Query.DESCENDING))
marker = [x for x in qry.Run()][0]
# Make sure we assigned the instance
self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk))
expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("One").hexdigest())
self.assertEqual(expected_marker, marker.key().id_or_name())
instance.name = "Two"
instance.save()
self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
marker = [x for x in qry.Run()][0]
# Make sure we assigned the instance
self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk))
expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("Two").hexdigest())
self.assertEqual(expected_marker, marker.key().id_or_name())
def test_repair_missing_markers_on_non_default_namespace(self):
self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1)
self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2)
NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"]
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE)
instance_key_ns1 = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=NS1_NAMESPACE)
marker = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key_default = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=DEFAULT_NAMESPACE)
marker_key_ns1 = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=NS1_NAMESPACE)
datastore.Delete(marker_key_ns1)
datastore.Delete(marker_key_default)
UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel), db="ns1")
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
# Is the missing marker for the default namespace left alone?
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, marker_key_default)
# Is the missing marker restored?
marker = datastore.Get(marker_key_ns1)
self.assertTrue(marker)
self.assertTrue(isinstance(marker["instance"], datastore.Key))
self.assertEqual(instance_key_ns1, marker["instance"])
self.assertTrue(marker["created"])
def test_clean_removes_markers_with_different_values(self):
marker1 = "{}|name:{}".format(TestModel._meta.db_table,
md5(self.i1.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(),
marker1,
namespace=DEFAULT_NAMESPACE)
original_marker = datastore.Get(marker_key)
marker2 = "{}|name:{}".format(TestModel._meta.db_table,
md5("bananas").hexdigest())
new_marker = datastore.Entity(UniqueMarker.kind(),
name=marker2,
namespace=DEFAULT_NAMESPACE)
new_marker.update(original_marker)
datastore.Put(new_marker)
UniqueAction.objects.create(action_type="clean",
model=encode_model(TestModel))
process_task_queues()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get,
new_marker.key())
self.assertTrue(datastore.Get(marker_key))
def test_repair_missing_markers_on_non_default_namespace(self):
self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1)
self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2)
NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"]
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE)
instance_key_ns1 = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=NS1_NAMESPACE)
marker = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key_default = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=DEFAULT_NAMESPACE)
marker_key_ns1 = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=NS1_NAMESPACE)
datastore.Delete(marker_key_ns1)
datastore.Delete(marker_key_default)
UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel), db="ns1")
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
# Is the missing marker for the default namespace left alone?
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, marker_key_default)
# Is the missing marker restored?
marker = datastore.Get(marker_key_ns1)
self.assertTrue(marker)
self.assertTrue(isinstance(marker["instance"], datastore.Key))
self.assertEqual(instance_key_ns1, marker["instance"])
self.assertTrue(marker["created"])
def test_clean_removes_markers_with_different_values_on_non_default_namespace(self):
self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1)
self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2)
NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"]
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i3.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=NS1_NAMESPACE)
default_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
original_marker = datastore.Get(marker_key)
default_marker = datastore.Get(default_key)
marker2 = "{}|name:{}".format(TestModel._meta.db_table, md5("bananas").hexdigest())
new_marker = datastore.Entity(UniqueMarker.kind(), name=marker2, namespace=NS1_NAMESPACE)
new_marker.update(original_marker)
datastore.Put(new_marker)
# This allows us to test: 1) namespaced markers will check against their namespace models (not all of them)"
self.i1.delete()
#... 2) the mapper only cleans the desired namespace
datastore.Put(default_marker)
UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel), db="ns1")
process_task_queues()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, new_marker.key())
self.assertTrue(datastore.Get(default_marker.key()))
self.assertTrue(datastore.Get(marker_key))
datastore.Delete(default_marker)
def execute(self):
table = self.table
query = datastore.Query(table,
keys_only=True,
namespace=self.namespace)
while query.Count():
datastore.Delete(query.Run())
# Delete the markers we need to
from djangae.db.constraints import UniqueMarker
query = datastore.Query(UniqueMarker.kind(),
keys_only=True,
namespace=self.namespace)
query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(),
self.table,
namespace=self.namespace)
query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(),
u"{}{}".format(
self.table,
u'\ufffd'),
namespace=self.namespace)
while query.Count():
datastore.Delete(query.Run())
# TODO: ideally we would only clear the cached objects for the table that was flushed, but
# we have no way of doing that
memcache.flush_all()
caching.get_context().reset()
def map(instance, *args, **kwargs):
""" Figure out what markers the instance should use and verify they're attached to
this instance. Log any weirdness and in repair mode - recreate missing markers. """
action_id = kwargs.get("action_pk")
repair = kwargs.get("repair")
alias = kwargs.get("db", "default")
namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE")
assert alias == (instance._state.db or "default")
entity, _ = django_instance_to_entities(connections[alias], instance._meta.fields, raw=True, instance=instance, check_null=False)
identifiers = unique_identifiers_from_entity(type(instance), entity, ignore_pk=True)
identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=namespace) for i in identifiers]
markers = datastore.Get(identifier_keys)
instance_key = str(entity.key())
markers_to_save = []
for i, m in zip(identifier_keys, markers):
marker_key = str(i)
if m is None:
# Missig marker
if repair:
new_marker = datastore.Entity(UniqueMarker.kind(), name=i.name(), namespace=namespace)
new_marker['instance'] = entity.key()
new_marker['created'] = datetime.datetime.now()
markers_to_save.append(new_marker)
else:
log(action_id, "missing_marker", instance_key, marker_key)
elif 'instance' not in m or not m['instance']:
# Marker with missining instance attribute
if repair:
m['instance'] = entity.key()
markers_to_save.append(m)
else:
log(action_id, "missing_instance", instance_key, marker_key)
elif m['instance'] != entity.key():
if isinstance(m['instance'], basestring):
m['instance'] = datastore.Key(m['instance'])
if repair:
markers_to_save.append(m)
else:
log(action_id, "old_instance_key", instance_key, marker_key)
if m['instance'] != entity.key():
# Marker already assigned to a different instance
log(action_id, "already_assigned", instance_key, marker_key)
# Also log in repair mode as reparing would break the other instance.
if markers_to_save:
datastore.Put(markers_to_save)
def map(instance, *args, **kwargs):
""" Figure out what markers the instance should use and verify they're attached to
this instance. Log any weirdness and in repair mode - recreate missing markers. """
action_id = kwargs.get("action_pk")
repair = kwargs.get("repair")
alias = kwargs.get("db", "default")
namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE")
assert alias == (instance._state.db or "default")
entity = django_instance_to_entity(connections[alias], type(instance), instance._meta.fields, raw=True, instance=instance, check_null=False)
identifiers = unique_identifiers_from_entity(type(instance), entity, ignore_pk=True)
identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=namespace) for i in identifiers]
markers = datastore.Get(identifier_keys)
instance_key = str(entity.key())
markers_to_save = []
for i, m in zip(identifier_keys, markers):
marker_key = str(i)
if m is None:
# Missig marker
if repair:
new_marker = datastore.Entity(UniqueMarker.kind(), name=i.name(), namespace=namespace)
new_marker['instance'] = entity.key()
new_marker['created'] = datetime.datetime.now()
markers_to_save.append(new_marker)
else:
log(action_id, "missing_marker", instance_key, marker_key)
elif 'instance' not in m or not m['instance']:
# Marker with missining instance attribute
if repair:
m['instance'] = entity.key()
markers_to_save.append(m)
else:
log(action_id, "missing_instance", instance_key, marker_key)
elif m['instance'] != entity.key():
if isinstance(m['instance'], basestring):
m['instance'] = datastore.Key(m['instance'])
if repair:
markers_to_save.append(m)
else:
log(action_id, "old_instance_key", instance_key, marker_key)
if m['instance'] != entity.key():
# Marker already assigned to a different instance
log(action_id, "already_assigned", instance_key, marker_key)
# Also log in repair mode as reparing would break the other instance.
if markers_to_save:
datastore.Put(markers_to_save)
def test_constraints_disabled_doesnt_create_or_check_markers(self):
initial_count = datastore.Query(UniqueMarker.kind()).Count()
instance1 = ModelWithUniques.objects.create(name="One")
self.assertEqual(initial_count, datastore.Query(UniqueMarker.kind()).Count())
instance2 = ModelWithUniques.objects.create(name="One")
self.assertEqual(instance1.name, instance2.name)
self.assertFalse(instance1 == instance2)
def func(*args, **kwargs):
kind = args[0][0].kind() if isinstance(args[0], list) else args[0].kind()
if kind == UniqueMarker.kind():
return original(*args, **kwargs)
raise AssertionError()
def map(entity, model, *args, **kwargs):
""" The Clean mapper maps over all UniqueMarker instances. """
alias = kwargs.get("db", "default")
namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE", "")
model = decode_model(model)
if not entity.key().id_or_name().startswith(model._meta.db_table + "|"):
# Only include markers which are for this model
return
assert namespace == entity.namespace()
with disable_cache():
# At this point, the entity is a unique marker that is linked to an instance of 'model', now we should see if that instance exists!
instance_id = entity["instance"].id_or_name()
try:
instance = model.objects.using(alias).get(pk=instance_id)
except model.DoesNotExist:
logger.info("Deleting unique marker %s because the associated instance no longer exists", entity.key().id_or_name())
datastore.Delete(entity)
return
# Get the possible unique markers for the entity, if this one doesn't exist in that list then delete it
instance_entity, _ = django_instance_to_entities(connections[alias], instance._meta.fields, raw=True, instance=instance, check_null=False)
identifiers = unique_identifiers_from_entity(model, instance_entity, ignore_pk=True)
identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=entity["instance"].namespace()) for i in identifiers]
if entity.key() not in identifier_keys:
logger.info("Deleting unique marker %s because the it no longer represents the associated instance state", entity.key().id_or_name())
datastore.Delete(entity)
def test_check_old_style_marker(self):
instance_key = datastore.Key.from_path(TestModel._meta.db_table,
self.i2.pk,
namespace=DEFAULT_NAMESPACE)
marker1 = "{}|name:{}".format(TestModel._meta.db_table,
md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(),
marker1,
namespace=DEFAULT_NAMESPACE)
marker = datastore.Get(marker_key)
marker['instance'] = str(instance_key) #Make the instance a string
datastore.Put(marker)
UniqueAction.objects.create(action_type="check",
model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(1, a.actionlog_set.count())
error = a.actionlog_set.all()[0]
self.assertEqual(error.log_type, "old_instance_key")
self.assertEqual(error.instance_key, str(instance_key))
self.assertEqual(error.marker_key, str(marker_key))
def map(entity, model, *args, **kwargs):
""" The Clean mapper maps over all UniqueMarker instances. """
alias = kwargs.get("db", "default")
namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE")
model = decode_model(model)
if not entity.key().id_or_name().startswith(model._meta.db_table + "|"):
# Only include markers which are for this model
return
assert namespace == entity.namespace()
with disable_cache():
# At this point, the entity is a unique marker that is linked to an instance of 'model', now we should see if that instance exists!
instance_id = entity["instance"].id_or_name()
try:
instance = model.objects.using(alias).get(pk=instance_id)
except model.DoesNotExist:
logging.info("Deleting unique marker {} because the associated instance no longer exists".format(entity.key().id_or_name()))
datastore.Delete(entity)
return
# Get the possible unique markers for the entity, if this one doesn't exist in that list then delete it
instance_entity = django_instance_to_entity(connections[alias], model, instance._meta.fields, raw=True, instance=instance, check_null=False)
identifiers = unique_identifiers_from_entity(model, instance_entity, ignore_pk=True)
identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=entity["instance"].namespace()) for i in identifiers]
if entity.key() not in identifier_keys:
logging.info("Deleting unique marker {} because the it no longer represents the associated instance state".format(entity.key().id_or_name()))
datastore.Delete(entity)
def execute(self):
table = self.table
query = datastore.Query(table, keys_only=True)
while query.Count():
datastore.Delete(query.Run())
# Delete the markers we need to
from djangae.db.constraints import UniqueMarker
query = datastore.Query(UniqueMarker.kind(), keys_only=True)
query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table)
query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(), u"{}{}".format(self.table, u'\ufffd'))
while query.Count():
datastore.Delete(query.Run())
cache.clear()
clear_context_cache()
def test_error_on_update_doesnt_change_markers(self):
initial_count = datastore.Query(UniqueMarker.kind()).Count()
instance = ModelWithUniques.objects.create(name="One")
self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
qry = datastore.Query(UniqueMarker.kind())
qry.Order(("created", datastore.Query.DESCENDING))
marker = [ x for x in qry.Run()][0]
# Make sure we assigned the instance
self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk))
expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("One").hexdigest())
self.assertEqual(expected_marker, marker.key().id_or_name())
instance.name = "Two"
from djangae.db.backends.appengine.commands import datastore as to_patch
try:
original = to_patch.Put
def func(*args, **kwargs):
kind = args[0][0].kind() if isinstance(args[0], list) else args[0].kind()
if kind == UniqueMarker.kind():
return original(*args, **kwargs)
raise AssertionError()
to_patch.Put = func
with self.assertRaises(Exception):
instance.save()
finally:
to_patch.Put = original
self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
marker = [x for x in qry.Run()][0]
# Make sure we assigned the instance
self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk))
expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("One").hexdigest())
self.assertEqual(expected_marker, marker.key().id_or_name())
def test_clean_removes_markers_with_different_values(self):
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i1.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
original_marker = datastore.Get(marker_key)
marker2 = "{}|name:{}".format(TestModel._meta.db_table, md5("bananas").hexdigest())
new_marker = datastore.Entity(UniqueMarker.kind(), name=marker2, namespace=DEFAULT_NAMESPACE)
new_marker.update(original_marker)
datastore.Put(new_marker)
UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel))
process_task_queues()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, new_marker.key())
self.assertTrue(datastore.Get(marker_key))
def execute(self):
table = self.table
query = datastore.Query(table, keys_only=True, namespace=self.namespace)
while query.Count():
datastore.Delete(query.Run())
# Delete the markers we need to
from djangae.db.constraints import UniqueMarker
query = datastore.Query(UniqueMarker.kind(), keys_only=True, namespace=self.namespace)
query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table, namespace=self.namespace)
query["__key__ <"] = datastore.Key.from_path(
UniqueMarker.kind(), u"{}{}".format(self.table, u'\ufffd'), namespace=self.namespace
)
while query.Count():
datastore.Delete(query.Run())
# TODO: ideally we would only clear the cached objects for the table that was flushed, but
# we have no way of doing that
memcache.flush_all()
caching.get_context().reset()
def test_clean_after_instance_deleted(self):
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i1.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
self.assertTrue(datastore.Get(marker_key))
datastore.Delete(datastore.Key.from_path(TestModel._meta.db_table, self.i1.pk, namespace=DEFAULT_NAMESPACE)) # Delete the first instance
self.assertTrue(datastore.Get(marker_key))
UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel))
process_task_queues()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, marker_key)
def test_clean_after_instance_deleted(self):
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i1.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
self.assertTrue(datastore.Get(marker_key))
datastore.Delete(datastore.Key.from_path(TestModel._meta.db_table, self.i1.pk, namespace=DEFAULT_NAMESPACE)) # Delete the first instance
self.assertTrue(datastore.Get(marker_key))
UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel))
process_task_queues()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, marker_key)
def test_error_on_insert_doesnt_create_markers(self):
initial_count = datastore.Query(UniqueMarker.kind()).Count()
from djangae.db.backends.appengine.commands import datastore as to_patch
try:
original = to_patch.Put
def func(*args, **kwargs):
kind = args[0][0].kind() if isinstance(args[0], list) else args[0].kind()
if kind == UniqueMarker.kind():
return original(*args, **kwargs)
raise AssertionError()
to_patch.Put = func
with self.assertRaises(AssertionError):
ModelWithUniques.objects.create(name="One")
finally:
to_patch.Put = original
self.assertEqual(0, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
def test_check_missing_markers(self):
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1)
datastore.Delete(marker_key)
UniqueAction.objects.create(action_type="check", model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(1, a.actionlog_set.count())
error = a.actionlog_set.all()[0]
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk)
self.assertEqual(error.log_type, "missing_marker")
self.assertEqual(error.instance_key, str(instance_key))
self.assertEqual(error.marker_key, str(marker_key))
def test_check_missing_markers(self):
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
datastore.Delete(marker_key)
UniqueAction.objects.create(action_type="check", model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(1, a.actionlog_set.count())
error = a.actionlog_set.all()[0]
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE)
self.assertEqual(error.log_type, "missing_marker")
self.assertEqual(error.instance_key, str(instance_key))
self.assertEqual(error.marker_key, str(marker_key))
def test_repair_missing_instance_attr(self):
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk)
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1)
marker = datastore.Get(marker_key)
marker['instance'] = None
datastore.Put(marker)
UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(0, a.actionlog_set.count())
marker = datastore.Get(marker_key)
self.assertTrue(marker)
self.assertEqual(marker['instance'], instance_key)
def test_repair_missing_markers(self):
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk)
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1)
datastore.Delete(marker_key)
UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(0, a.actionlog_set.count())
# Is the missing marker restored?
marker = datastore.Get(marker_key)
self.assertTrue(marker)
self.assertTrue(isinstance(marker["instance"], datastore.Key))
self.assertEqual(instance_key, marker["instance"])
self.assertTrue(marker["created"])
def test_repair_missing_markers(self):
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE)
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
datastore.Delete(marker_key)
UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(0, a.actionlog_set.count())
# Is the missing marker restored?
marker = datastore.Get(marker_key)
self.assertTrue(marker)
self.assertTrue(isinstance(marker["instance"], datastore.Key))
self.assertEqual(instance_key, marker["instance"])
self.assertTrue(marker["created"])
def test_repair_old_style_marker(self):
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE)
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE)
marker = datastore.Get(marker_key)
marker['instance'] = str(instance_key) #Make the instance a string
datastore.Put(marker)
UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(0, a.actionlog_set.count())
marker = datastore.Get(marker_key)
self.assertTrue(marker)
self.assertEqual(marker['instance'], instance_key)
def test_check_old_style_marker(self):
instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk)
marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest())
marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1)
marker = datastore.Get(marker_key)
marker['instance'] = str(instance_key) #Make the instance a string
datastore.Put(marker)
UniqueAction.objects.create(action_type="check", model=encode_model(TestModel))
process_task_queues()
a = UniqueAction.objects.get()
self.assertEqual(a.status, "done")
self.assertEqual(1, a.actionlog_set.count())
error = a.actionlog_set.all()[0]
self.assertEqual(error.log_type, "old_instance_key")
self.assertEqual(error.instance_key, str(instance_key))
self.assertEqual(error.marker_key, str(marker_key))
def map(instance, *args, **kwargs):
""" Figure out what markers the instance should use and verify they're attached to
this instance. Log any weirdness and in repair mode - recreate missing markers. """
action_id = kwargs.get("action_pk")
repair = kwargs.get("repair")
entity = django_instance_to_entity(connection,
type(instance),
instance._meta.fields,
raw=False,
instance=instance)
identifiers = unique_identifiers_from_entity(type(instance),
entity,
ignore_pk=True)
identifier_keys = [
datastore.Key.from_path(UniqueMarker.kind(), i)
for i in identifiers
]
markers = datastore.Get(identifier_keys)
instance_key = str(entity.key())
markers_to_save = []
for i, m in zip(identifier_keys, markers):
marker_key = str(i)
if m is None:
# Missig marker
if repair:
new_marker = datastore.Entity(UniqueMarker.kind(),
name=i.name())
new_marker['instance'] = entity.key()
markers_to_save.append(new_marker)
else:
log(action_id, "missing_marker", instance_key, marker_key)
elif 'instance' not in m or not m['instance']:
# Marker with missining instance attribute
if repair:
m['instance'] = entity.key()
markers_to_save.append(m)
else:
log(action_id, "missing_instance", instance_key,
marker_key)
elif m['instance'] != entity.key():
if isinstance(m['instance'], basestring):
m['instance'] = datastore.Key(m['instance'])
if repair:
markers_to_save.append(m)
else:
log(action_id, "old_instance_key", instance_key,
marker_key)
if m['instance'] != entity.key():
# Marker already assigned to a different instance
log(action_id, "already_assigned", instance_key,
marker_key)
# Also log in repair mode as reparing would break the other instance.
if markers_to_save:
datastore.Put(markers_to_save)
yield ('_', [instance.pk])
