def test_uid_allows_separate_identical_operations_to_be_run(self):
""" By passing the 'uid' kwarg to an operation, we should allow it to be run, even if an
otherwise idential operation has already been run.
"""
operation1 = operations.AddFieldData("testmodel", "new_field",
models.BooleanField(default=True))
operation2 = operations.AddFieldData("testmodel", "new_field",
models.BooleanField(default=True))
operation3 = operations.AddFieldData("testmodel",
"new_field",
models.BooleanField(default=True),
uid="x")
# Create a model instance and run the first operation on it
instance = TestModel.objects.create()
self.start_operation(operation1)
self.process_task_queues()
# Check that the migration ran successfully
entity = self.get_entities()[0]
self.assertTrue(entity["new_field"])
# Now create another entity and make sure that the second migration (which is idential)
# does NOT run on it
instance.delete()
instance = TestModel.objects.create()
self.start_operation(operation2)
self.process_task_queues()
entity = self.get_entities()[0]
self.assertIsNone(entity.get("new_field"))
# Now run the third operation, which is identical but has a uid, so SHOULD be run
self.start_operation(operation3)
self.process_task_queues()
entity = self.get_entities()[0]
self.assertTrue(entity["new_field"])
def test_default_queue_setting(self):
""" If no `queue` kwarg is passed then the DJANGAE_MIGRATION_DEFAULT_QUEUE setting should
be used to determine the task queue.
"""
for x in range(2):
TestModel.objects.create()
operation = operations.AddFieldData(
"testmodel",
"new_field",
models.CharField(max_length=100, default="squirrel"),
)
# Check that starting the operation with a different setting correctly affects the queue.
# Note that here we don't check that *all* tasks go on the correct queue, just the first
# one. We test that more thoroughly in `test_queue_option` above.
with override_settings(DJANGAE_MIGRATION_DEFAULT_QUEUE="another"):
self.start_operation(operation)
self.assertEqual(self.get_task_count("default"), 0)
self.assertTrue(self.get_task_count("another") > 0)
self.flush_task_queues()
flush_task_markers()
# santity checks:
assert getattr(settings, "DJANGAE_MIGRATION_DEFAULT_QUEUE",
None) is None
assert self.get_task_count() == 0
# Trigger the operation without that setting. The task(s) should go on the default queue.
self.start_operation(operation)
self.assertTrue(self.get_task_count("default") > 0)
def test_run_operation_creates_and_updates_task_marker(self):
""" If we run one of our custom operations, then it should create the task marker in the DB
and defer a task, then set the marker to 'is_finished' when done.
"""
TestModel.objects.create()
operation = operations.AddFieldData(
"testmodel", "new_field",
models.CharField(max_length=100, default="squirrel"))
self.start_operation(operation)
# Now check that the task marker has been created.
# Usefully, calling database_forwards() on the operation will have caused it to set the
# `identifier` attribute on itself, meaning we can now just call _get_task_marker()
task_marker = datastore.Get([
ShardedTaskMarker.get_key(operation.identifier,
operation.namespace)
])[0]
if task_marker is None:
self.fail("Migration operation did not create its task marker")
self.assertFalse(task_marker.get("is_finished"))
self.assertNumTasksEquals(1)
self.process_task_queues()
# Now check that the task marker has been marked as finished
task_marker = datastore.Get([
ShardedTaskMarker.get_key(operation.identifier,
operation.namespace)
])[0]
self.assertTrue(task_marker["is_finished"])
self.assertNumTasksEquals(0)
def test_running_finished_operation_does_not_trigger_new_task(self):
""" If we re-trigger an operation which has already been run and finished, it should simply
return without starting a new task or updating the task marker.
"""
TestModel.objects.create()
operation = operations.AddFieldData(
"testmodel", "new_field", models.CharField(max_length=100, default="squirrel")
)
# Run the operation and check that it finishes
with sleuth.watch("djangae.db.migrations.operations.AddFieldData._start_task") as start:
self.start_operation(operation)
self.assertTrue(start.called)
task_marker = datastore.Get(
ShardedTaskMarker.get_key(operation.identifier, operation.namespace)
)
self.assertFalse(task_marker["is_finished"])
self.assertNumTasksEquals(1)
self.process_task_queues()
task_marker = datastore.Get(
ShardedTaskMarker.get_key(operation.identifier, operation.namespace)
)
self.assertTrue(task_marker["is_finished"])
# Run the operation again. It should see that's it's finished and just return immediately.
self.assertNumTasksEquals(0)
with sleuth.watch("djangae.db.migrations.operations.AddFieldData._start_task") as start:
self.start_operation(operation, detonate=False)
self.assertFalse(start.called)
self.assertNumTasksEquals(0)
task_marker = datastore.Get(
ShardedTaskMarker.get_key(operation.identifier, operation.namespace)
)
self.assertTrue(task_marker["is_finished"])
def test_addfielddata(self):
""" Test the AddFieldData operation. """
for x in range(2):
TestModel.objects.create()
# Just for sanity, check that none of the entities have the new field value yet
entities = self.get_entities()
self.assertFalse(any(entity.get("new_field") for entity in entities))
operation = operations.AddFieldData(
"testmodel", "new_field", models.CharField(max_length=100, default="squirrel")
)
self.start_operation(operation)
self.process_task_queues()
# The entities should now all have the 'new_field' actually mapped over
entities = self.get_entities()
self.assertTrue(all(entity['new_field'] == 'squirrel' for entity in entities))
def test_starting_operation_twice_does_not_trigger_task_twice(self):
""" If we run an operation, and then try to run it again before the task has finished
processing, then it should not trigger a second task.
"""
TestModel.objects.create()
operation = operations.AddFieldData(
"testmodel", "new_field",
models.CharField(max_length=100, default="squirrel"))
self.start_operation(operation)
task_marker = datastore.Get(
ShardedTaskMarker.get_key(operation.identifier,
operation.namespace))
self.assertFalse(task_marker["is_finished"])
# We expect there to be a task queued for processing the operation
self.assertNumTasksEquals(1)
# Now try to run it again
self.start_operation(operation)
# We expect there to still be the same number of tasks
self.assertNumTasksEquals(1)
def test_queue_option(self):
""" The `queue` kwarg should determine the task queue that the operation runs on. """
for x in range(3):
TestModel.objects.create()
operation = operations.AddFieldData(
"testmodel", "new_field", models.CharField(max_length=100, default=return_a_string),
queue="another",
# Ensure that we trigger a re-defer, so that we test that the correct queue is used for
# subsequent tasks, not just the first one
entities_per_task=1,
shard_count=1
)
self.start_operation(operation)
# The task(s) should not be in the default queue, but in the "another" queue instead
self.assertEqual(self.get_task_count("default"), 0)
self.assertTrue(self.get_task_count("another") > 0)
# And if we only run the tasks on the "another" queue, the whole operation should complete.
self.process_task_queues("another")
# And the entities should be updated
entities = self.get_entities()
self.assertTrue(all(entity['new_field'] == 'squirrel' for entity in entities))