def test_clean_removes_markers_with_different_values_on_non_default_namespace(self): self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1) self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2) NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"] marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i3.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=NS1_NAMESPACE) default_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) original_marker = datastore.Get(marker_key) default_marker = datastore.Get(default_key) marker2 = "{}|name:{}".format(TestModel._meta.db_table, md5("bananas").hexdigest()) new_marker = datastore.Entity(UniqueMarker.kind(), name=marker2, namespace=NS1_NAMESPACE) new_marker.update(original_marker) datastore.Put(new_marker) # This allows us to test: 1) namespaced markers will check against their namespace models (not all of them)" self.i1.delete() #... 2) the mapper only cleans the desired namespace datastore.Put(default_marker) UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel), db="ns1") process_task_queues() self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, new_marker.key()) self.assertTrue(datastore.Get(default_marker.key())) self.assertTrue(datastore.Get(marker_key)) datastore.Delete(default_marker)
def test_constraints_can_be_enabled_per_model(self): initial_count = datastore.Query(UniqueMarker.kind()).Count() instance1 = ModelWithUniquesAndOverride.objects.create(name="One") self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
def test_delete_clears_markers(self): initial_count = datastore.Query(UniqueMarker.kind()).Count() instance = ModelWithUniques.objects.create(name="One") self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count) instance.delete() self.assertEqual(0, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
def test_update_updates_markers(self): initial_count = datastore.Query(UniqueMarker.kind()).Count() instance = ModelWithUniques.objects.create(name="One") self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count) qry = datastore.Query(UniqueMarker.kind()) qry.Order(("created", datastore.Query.DESCENDING)) marker = [x for x in qry.Run()][0] # Make sure we assigned the instance self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk)) expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("One").hexdigest()) self.assertEqual(expected_marker, marker.key().id_or_name()) instance.name = "Two" instance.save() self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count) marker = [x for x in qry.Run()][0] # Make sure we assigned the instance self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk)) expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("Two").hexdigest()) self.assertEqual(expected_marker, marker.key().id_or_name())
def test_repair_missing_markers_on_non_default_namespace(self): self.i3 = TestModel.objects.using("ns1").create(id=self.i1.pk, name="name1", counter1=1, counter2=1) self.i4 = TestModel.objects.using("ns1").create(id=self.i2.pk, name="name3", counter1=1, counter2=2) NS1_NAMESPACE = settings.DATABASES["ns1"]["NAMESPACE"] instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE) instance_key_ns1 = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=NS1_NAMESPACE) marker = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key_default = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=DEFAULT_NAMESPACE) marker_key_ns1 = datastore.Key.from_path(UniqueMarker.kind(), marker, namespace=NS1_NAMESPACE) datastore.Delete(marker_key_ns1) datastore.Delete(marker_key_default) UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel), db="ns1") process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") # Is the missing marker for the default namespace left alone? self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, marker_key_default) # Is the missing marker restored? marker = datastore.Get(marker_key_ns1) self.assertTrue(marker) self.assertTrue(isinstance(marker["instance"], datastore.Key)) self.assertEqual(instance_key_ns1, marker["instance"]) self.assertTrue(marker["created"])
def test_clean_removes_markers_with_different_values(self): marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i1.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) original_marker = datastore.Get(marker_key) marker2 = "{}|name:{}".format(TestModel._meta.db_table, md5("bananas").hexdigest()) new_marker = datastore.Entity(UniqueMarker.kind(), name=marker2, namespace=DEFAULT_NAMESPACE) new_marker.update(original_marker) datastore.Put(new_marker) UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel)) process_task_queues() self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, new_marker.key()) self.assertTrue(datastore.Get(marker_key))
def execute(self): table = self.table query = datastore.Query(table, keys_only=True, namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # Delete the markers we need to from djangae.db.constraints import UniqueMarker query = datastore.Query(UniqueMarker.kind(), keys_only=True, namespace=self.namespace) query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table, namespace=self.namespace) query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(), u"{}{}".format( self.table, u'\ufffd'), namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # TODO: ideally we would only clear the cached objects for the table that was flushed, but # we have no way of doing that memcache.flush_all() caching.get_context().reset()
def map(instance, *args, **kwargs): """ Figure out what markers the instance should use and verify they're attached to this instance. Log any weirdness and in repair mode - recreate missing markers. """ action_id = kwargs.get("action_pk") repair = kwargs.get("repair") alias = kwargs.get("db", "default") namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE") assert alias == (instance._state.db or "default") entity, _ = django_instance_to_entities(connections[alias], instance._meta.fields, raw=True, instance=instance, check_null=False) identifiers = unique_identifiers_from_entity(type(instance), entity, ignore_pk=True) identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=namespace) for i in identifiers] markers = datastore.Get(identifier_keys) instance_key = str(entity.key()) markers_to_save = [] for i, m in zip(identifier_keys, markers): marker_key = str(i) if m is None: # Missig marker if repair: new_marker = datastore.Entity(UniqueMarker.kind(), name=i.name(), namespace=namespace) new_marker['instance'] = entity.key() new_marker['created'] = datetime.datetime.now() markers_to_save.append(new_marker) else: log(action_id, "missing_marker", instance_key, marker_key) elif 'instance' not in m or not m['instance']: # Marker with missining instance attribute if repair: m['instance'] = entity.key() markers_to_save.append(m) else: log(action_id, "missing_instance", instance_key, marker_key) elif m['instance'] != entity.key(): if isinstance(m['instance'], basestring): m['instance'] = datastore.Key(m['instance']) if repair: markers_to_save.append(m) else: log(action_id, "old_instance_key", instance_key, marker_key) if m['instance'] != entity.key(): # Marker already assigned to a different instance log(action_id, "already_assigned", instance_key, marker_key) # Also log in repair mode as reparing would break the other instance. if markers_to_save: datastore.Put(markers_to_save)
def map(instance, *args, **kwargs): """ Figure out what markers the instance should use and verify they're attached to this instance. Log any weirdness and in repair mode - recreate missing markers. """ action_id = kwargs.get("action_pk") repair = kwargs.get("repair") alias = kwargs.get("db", "default") namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE") assert alias == (instance._state.db or "default") entity = django_instance_to_entity(connections[alias], type(instance), instance._meta.fields, raw=True, instance=instance, check_null=False) identifiers = unique_identifiers_from_entity(type(instance), entity, ignore_pk=True) identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=namespace) for i in identifiers] markers = datastore.Get(identifier_keys) instance_key = str(entity.key()) markers_to_save = [] for i, m in zip(identifier_keys, markers): marker_key = str(i) if m is None: # Missig marker if repair: new_marker = datastore.Entity(UniqueMarker.kind(), name=i.name(), namespace=namespace) new_marker['instance'] = entity.key() new_marker['created'] = datetime.datetime.now() markers_to_save.append(new_marker) else: log(action_id, "missing_marker", instance_key, marker_key) elif 'instance' not in m or not m['instance']: # Marker with missining instance attribute if repair: m['instance'] = entity.key() markers_to_save.append(m) else: log(action_id, "missing_instance", instance_key, marker_key) elif m['instance'] != entity.key(): if isinstance(m['instance'], basestring): m['instance'] = datastore.Key(m['instance']) if repair: markers_to_save.append(m) else: log(action_id, "old_instance_key", instance_key, marker_key) if m['instance'] != entity.key(): # Marker already assigned to a different instance log(action_id, "already_assigned", instance_key, marker_key) # Also log in repair mode as reparing would break the other instance. if markers_to_save: datastore.Put(markers_to_save)
def test_constraints_disabled_doesnt_create_or_check_markers(self): initial_count = datastore.Query(UniqueMarker.kind()).Count() instance1 = ModelWithUniques.objects.create(name="One") self.assertEqual(initial_count, datastore.Query(UniqueMarker.kind()).Count()) instance2 = ModelWithUniques.objects.create(name="One") self.assertEqual(instance1.name, instance2.name) self.assertFalse(instance1 == instance2)
def func(*args, **kwargs): kind = args[0][0].kind() if isinstance(args[0], list) else args[0].kind() if kind == UniqueMarker.kind(): return original(*args, **kwargs) raise AssertionError()
def map(entity, model, *args, **kwargs): """ The Clean mapper maps over all UniqueMarker instances. """ alias = kwargs.get("db", "default") namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE", "") model = decode_model(model) if not entity.key().id_or_name().startswith(model._meta.db_table + "|"): # Only include markers which are for this model return assert namespace == entity.namespace() with disable_cache(): # At this point, the entity is a unique marker that is linked to an instance of 'model', now we should see if that instance exists! instance_id = entity["instance"].id_or_name() try: instance = model.objects.using(alias).get(pk=instance_id) except model.DoesNotExist: logger.info("Deleting unique marker %s because the associated instance no longer exists", entity.key().id_or_name()) datastore.Delete(entity) return # Get the possible unique markers for the entity, if this one doesn't exist in that list then delete it instance_entity, _ = django_instance_to_entities(connections[alias], instance._meta.fields, raw=True, instance=instance, check_null=False) identifiers = unique_identifiers_from_entity(model, instance_entity, ignore_pk=True) identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=entity["instance"].namespace()) for i in identifiers] if entity.key() not in identifier_keys: logger.info("Deleting unique marker %s because the it no longer represents the associated instance state", entity.key().id_or_name()) datastore.Delete(entity)
def test_check_old_style_marker(self): instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE) marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) marker = datastore.Get(marker_key) marker['instance'] = str(instance_key) #Make the instance a string datastore.Put(marker) UniqueAction.objects.create(action_type="check", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(1, a.actionlog_set.count()) error = a.actionlog_set.all()[0] self.assertEqual(error.log_type, "old_instance_key") self.assertEqual(error.instance_key, str(instance_key)) self.assertEqual(error.marker_key, str(marker_key))
def map(entity, model, *args, **kwargs): """ The Clean mapper maps over all UniqueMarker instances. """ alias = kwargs.get("db", "default") namespace = settings.DATABASES.get(alias, {}).get("NAMESPACE") model = decode_model(model) if not entity.key().id_or_name().startswith(model._meta.db_table + "|"): # Only include markers which are for this model return assert namespace == entity.namespace() with disable_cache(): # At this point, the entity is a unique marker that is linked to an instance of 'model', now we should see if that instance exists! instance_id = entity["instance"].id_or_name() try: instance = model.objects.using(alias).get(pk=instance_id) except model.DoesNotExist: logging.info("Deleting unique marker {} because the associated instance no longer exists".format(entity.key().id_or_name())) datastore.Delete(entity) return # Get the possible unique markers for the entity, if this one doesn't exist in that list then delete it instance_entity = django_instance_to_entity(connections[alias], model, instance._meta.fields, raw=True, instance=instance, check_null=False) identifiers = unique_identifiers_from_entity(model, instance_entity, ignore_pk=True) identifier_keys = [datastore.Key.from_path(UniqueMarker.kind(), i, namespace=entity["instance"].namespace()) for i in identifiers] if entity.key() not in identifier_keys: logging.info("Deleting unique marker {} because the it no longer represents the associated instance state".format(entity.key().id_or_name())) datastore.Delete(entity)
def execute(self): table = self.table query = datastore.Query(table, keys_only=True) while query.Count(): datastore.Delete(query.Run()) # Delete the markers we need to from djangae.db.constraints import UniqueMarker query = datastore.Query(UniqueMarker.kind(), keys_only=True) query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table) query["__key__ <"] = datastore.Key.from_path(UniqueMarker.kind(), u"{}{}".format(self.table, u'\ufffd')) while query.Count(): datastore.Delete(query.Run()) cache.clear() clear_context_cache()
def test_error_on_update_doesnt_change_markers(self): initial_count = datastore.Query(UniqueMarker.kind()).Count() instance = ModelWithUniques.objects.create(name="One") self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count) qry = datastore.Query(UniqueMarker.kind()) qry.Order(("created", datastore.Query.DESCENDING)) marker = [ x for x in qry.Run()][0] # Make sure we assigned the instance self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk)) expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("One").hexdigest()) self.assertEqual(expected_marker, marker.key().id_or_name()) instance.name = "Two" from djangae.db.backends.appengine.commands import datastore as to_patch try: original = to_patch.Put def func(*args, **kwargs): kind = args[0][0].kind() if isinstance(args[0], list) else args[0].kind() if kind == UniqueMarker.kind(): return original(*args, **kwargs) raise AssertionError() to_patch.Put = func with self.assertRaises(Exception): instance.save() finally: to_patch.Put = original self.assertEqual(1, datastore.Query(UniqueMarker.kind()).Count() - initial_count) marker = [x for x in qry.Run()][0] # Make sure we assigned the instance self.assertEqual(datastore.Key(marker["instance"]), datastore.Key.from_path(instance._meta.db_table, instance.pk)) expected_marker = "{}|name:{}".format(ModelWithUniques._meta.db_table, md5("One").hexdigest()) self.assertEqual(expected_marker, marker.key().id_or_name())
def execute(self): table = self.table query = datastore.Query(table, keys_only=True, namespace=self.namespace) while query.Count(): datastore.Delete(query.Run()) # Delete the markers we need to from djangae.db.constraints import UniqueMarker query = datastore.Query(UniqueMarker.kind(), keys_only=True, namespace=self.namespace) query["__key__ >="] = datastore.Key.from_path(UniqueMarker.kind(), self.table, namespace=self.namespace) query["__key__ <"] = datastore.Key.from_path( UniqueMarker.kind(), u"{}{}".format(self.table, u'\ufffd'), namespace=self.namespace ) while query.Count(): datastore.Delete(query.Run()) # TODO: ideally we would only clear the cached objects for the table that was flushed, but # we have no way of doing that memcache.flush_all() caching.get_context().reset()
def test_clean_after_instance_deleted(self): marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i1.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) self.assertTrue(datastore.Get(marker_key)) datastore.Delete(datastore.Key.from_path(TestModel._meta.db_table, self.i1.pk, namespace=DEFAULT_NAMESPACE)) # Delete the first instance self.assertTrue(datastore.Get(marker_key)) UniqueAction.objects.create(action_type="clean", model=encode_model(TestModel)) process_task_queues() self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, marker_key)
def test_error_on_insert_doesnt_create_markers(self): initial_count = datastore.Query(UniqueMarker.kind()).Count() from djangae.db.backends.appengine.commands import datastore as to_patch try: original = to_patch.Put def func(*args, **kwargs): kind = args[0][0].kind() if isinstance(args[0], list) else args[0].kind() if kind == UniqueMarker.kind(): return original(*args, **kwargs) raise AssertionError() to_patch.Put = func with self.assertRaises(AssertionError): ModelWithUniques.objects.create(name="One") finally: to_patch.Put = original self.assertEqual(0, datastore.Query(UniqueMarker.kind()).Count() - initial_count)
def test_check_missing_markers(self): marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1) datastore.Delete(marker_key) UniqueAction.objects.create(action_type="check", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(1, a.actionlog_set.count()) error = a.actionlog_set.all()[0] instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk) self.assertEqual(error.log_type, "missing_marker") self.assertEqual(error.instance_key, str(instance_key)) self.assertEqual(error.marker_key, str(marker_key))
def test_check_missing_markers(self): marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) datastore.Delete(marker_key) UniqueAction.objects.create(action_type="check", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(1, a.actionlog_set.count()) error = a.actionlog_set.all()[0] instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE) self.assertEqual(error.log_type, "missing_marker") self.assertEqual(error.instance_key, str(instance_key)) self.assertEqual(error.marker_key, str(marker_key))
def test_repair_missing_instance_attr(self): instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk) marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1) marker = datastore.Get(marker_key) marker['instance'] = None datastore.Put(marker) UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(0, a.actionlog_set.count()) marker = datastore.Get(marker_key) self.assertTrue(marker) self.assertEqual(marker['instance'], instance_key)
def test_repair_missing_markers(self): instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk) marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1) datastore.Delete(marker_key) UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(0, a.actionlog_set.count()) # Is the missing marker restored? marker = datastore.Get(marker_key) self.assertTrue(marker) self.assertTrue(isinstance(marker["instance"], datastore.Key)) self.assertEqual(instance_key, marker["instance"]) self.assertTrue(marker["created"])
def test_repair_missing_markers(self): instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE) marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) datastore.Delete(marker_key) UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(0, a.actionlog_set.count()) # Is the missing marker restored? marker = datastore.Get(marker_key) self.assertTrue(marker) self.assertTrue(isinstance(marker["instance"], datastore.Key)) self.assertEqual(instance_key, marker["instance"]) self.assertTrue(marker["created"])
def test_repair_old_style_marker(self): instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk, namespace=DEFAULT_NAMESPACE) marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1, namespace=DEFAULT_NAMESPACE) marker = datastore.Get(marker_key) marker['instance'] = str(instance_key) #Make the instance a string datastore.Put(marker) UniqueAction.objects.create(action_type="repair", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(0, a.actionlog_set.count()) marker = datastore.Get(marker_key) self.assertTrue(marker) self.assertEqual(marker['instance'], instance_key)
def test_check_old_style_marker(self): instance_key = datastore.Key.from_path(TestModel._meta.db_table, self.i2.pk) marker1 = "{}|name:{}".format(TestModel._meta.db_table, md5(self.i2.name).hexdigest()) marker_key = datastore.Key.from_path(UniqueMarker.kind(), marker1) marker = datastore.Get(marker_key) marker['instance'] = str(instance_key) #Make the instance a string datastore.Put(marker) UniqueAction.objects.create(action_type="check", model=encode_model(TestModel)) process_task_queues() a = UniqueAction.objects.get() self.assertEqual(a.status, "done") self.assertEqual(1, a.actionlog_set.count()) error = a.actionlog_set.all()[0] self.assertEqual(error.log_type, "old_instance_key") self.assertEqual(error.instance_key, str(instance_key)) self.assertEqual(error.marker_key, str(marker_key))
def map(instance, *args, **kwargs): """ Figure out what markers the instance should use and verify they're attached to this instance. Log any weirdness and in repair mode - recreate missing markers. """ action_id = kwargs.get("action_pk") repair = kwargs.get("repair") entity = django_instance_to_entity(connection, type(instance), instance._meta.fields, raw=False, instance=instance) identifiers = unique_identifiers_from_entity(type(instance), entity, ignore_pk=True) identifier_keys = [ datastore.Key.from_path(UniqueMarker.kind(), i) for i in identifiers ] markers = datastore.Get(identifier_keys) instance_key = str(entity.key()) markers_to_save = [] for i, m in zip(identifier_keys, markers): marker_key = str(i) if m is None: # Missig marker if repair: new_marker = datastore.Entity(UniqueMarker.kind(), name=i.name()) new_marker['instance'] = entity.key() markers_to_save.append(new_marker) else: log(action_id, "missing_marker", instance_key, marker_key) elif 'instance' not in m or not m['instance']: # Marker with missining instance attribute if repair: m['instance'] = entity.key() markers_to_save.append(m) else: log(action_id, "missing_instance", instance_key, marker_key) elif m['instance'] != entity.key(): if isinstance(m['instance'], basestring): m['instance'] = datastore.Key(m['instance']) if repair: markers_to_save.append(m) else: log(action_id, "old_instance_key", instance_key, marker_key) if m['instance'] != entity.key(): # Marker already assigned to a different instance log(action_id, "already_assigned", instance_key, marker_key) # Also log in repair mode as reparing would break the other instance. if markers_to_save: datastore.Put(markers_to_save) yield ('_', [instance.pk])