def build_graph(self): """ Build a migrations dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migrations stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data if self.connection is None: self.applied_migrations = set() else: recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # To start, populate the migrations graph with nodes for ALL migrations # and their dependencies. Also make note of replacing migrations at this step. self.graph = MigrationGraph() self.replacements = {} for key, migration in self.disk_migrations.items(): self.graph.add_node(key, migration) # Replacing migrations. if migration.replaces: self.replacements[key] = migration for key, migration in self.disk_migrations.items(): # Internal (same app) dependencies. self.add_internal_dependencies(key, migration) # Add external dependencies now that the internal ones have been resolved. for key, migration in self.disk_migrations.items(): self.add_external_dependencies(key, migration) # Carry out replacements where possible. for key, migration in self.replacements.items(): # Get applied status of each of this migrations's replacement targets. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] # Ensure the replacing migrations is only marked as applied if all of # its replacement targets are. if all(applied_statuses): self.applied_migrations.add(key) else: self.applied_migrations.discard(key) # A replacing migrations can be used if either all or none of its # replacement targets have been applied. if all(applied_statuses) or (not any(applied_statuses)): self.graph.remove_replaced_nodes(key, migration.replaces) else: # This replacing migrations cannot be used because it is partially applied. # Remove it from the graph and remap dependencies to it (#25945). self.graph.remove_replacement_node(key, migration.replaces) # Ensure the graph is consistent. try: self.graph.validate_consistency() except NodeNotFoundError as exc: # Check if the missing node could have been replaced by any squash # migrations but wasn't because the squash migrations was partially # applied before. In that case raise a more understandable exception # (#23556). # Get reverse replacements. reverse_replacements = {} for key, migration in self.replacements.items(): for replaced in migration.replaces: reverse_replacements.setdefault(replaced, set()).add(key) # Try to reraise exception with more detail. if exc.node in reverse_replacements: candidates = reverse_replacements.get(exc.node, set()) is_replaced = any(candidate in self.graph.nodes for candidate in candidates) if not is_replaced: tries = ', '.join('%s.%s' % c for c in candidates) raise NodeNotFoundError( "Migration {0} depends on nonexistent node ('{1}', '{2}'). " "Django tried to replace migrations {1}.{2} with any of [{3}] " "but wasn't able to because some of the replaced migrations " "are already applied.".format(exc.origin, exc.node[0], exc.node[1], tries), exc.node) from exc raise exc self.graph.ensure_not_cyclic()
def build_graph(self): """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data if self.connection is None: self.applied_migrations = set() else: recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do a first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Carry out replacements if we can - that is, if all replaced migrations # are either unapplied or missing. for key, migration in replacing.items(): # Ensure this replacement migration is not in applied_migrations self.applied_migrations.discard(key) # Do the check. We can replace if all our replace targets are # applied, or if all of them are unapplied. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] can_replace = all(applied_statuses) or (not any(applied_statuses)) if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): if child_key in migration.replaces: continue normal[child_key].dependencies.remove(replaced) normal[child_key].dependencies.append(key) normal[key] = migration # Mark the replacement as applied if all its replaced ones are if all(applied_statuses): self.applied_migrations.add(key) # Finally, make a graph and load everything into it self.graph = MigrationGraph() for key, migration in normal.items(): self.graph.add_node(key, migration) # Add all internal dependencies first to ensure __first__ dependencies # find the correct root node. for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] != key[0] or parent[1] == '__first__': # Ignore __first__ references to the same app (#22325) continue self.graph.add_dependency(key, parent) for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] == key[0]: # Internal dependencies already added. continue parent = self.check_key(parent, key[0]) if parent is not None: self.graph.add_dependency(key, parent) for child in migration.run_before: child = self.check_key(child, key[0]) if child is not None: self.graph.add_dependency(child, key)
def build_graph(self): """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data if self.connection is None: self.applied_migrations = set() else: recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do a first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Remember the possible replacements to generate more meaningful error # messages reverse_replacements = {} for key, migration in replacing.items(): for replaced in migration.replaces: reverse_replacements.setdefault(replaced, set()).add(key) # Carry out replacements if we can - that is, if all replaced migrations # are either unapplied or missing. for key, migration in replacing.items(): # Ensure this replacement migration is not in applied_migrations self.applied_migrations.discard(key) # Do the check. We can replace if all our replace targets are # applied, or if all of them are unapplied. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] can_replace = all(applied_statuses) or (not any(applied_statuses)) if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): if child_key in migration.replaces: continue # child_key may appear in a replacement if child_key in reverse_replacements: for replaced_child_key in reverse_replacements[child_key]: if replaced in replacing[replaced_child_key].dependencies: replacing[replaced_child_key].dependencies.remove(replaced) replacing[replaced_child_key].dependencies.append(key) else: normal[child_key].dependencies.remove(replaced) normal[child_key].dependencies.append(key) normal[key] = migration # Mark the replacement as applied if all its replaced ones are if all(applied_statuses): self.applied_migrations.add(key) # Finally, make a graph and load everything into it self.graph = MigrationGraph() for key, migration in normal.items(): self.graph.add_node(key, migration) def _reraise_missing_dependency(migration, missing, exc): """ Checks if ``missing`` could have been replaced by any squash migration but wasn't because the the squash migration was partially applied before. In that case raise a more understandable exception. #23556 """ if missing in reverse_replacements: candidates = reverse_replacements.get(missing, set()) is_replaced = any(candidate in self.graph.nodes for candidate in candidates) if not is_replaced: tries = ', '.join('%s.%s' % c for c in candidates) exc_value = NodeNotFoundError( "Migration {0} depends on nonexistent node ('{1}', '{2}'). " "Django tried to replace migration {1}.{2} with any of [{3}] " "but wasn't able to because some of the replaced migrations " "are already applied.".format( migration, missing[0], missing[1], tries ), missing) exc_value.__cause__ = exc six.reraise(NodeNotFoundError, exc_value, sys.exc_info()[2]) raise exc # Add all internal dependencies first to ensure __first__ dependencies # find the correct root node. for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] != key[0] or parent[1] == '__first__': # Ignore __first__ references to the same app (#22325) continue try: self.graph.add_dependency(migration, key, parent) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "parent" is not in there. To make the raised exception # more understandable we check if parent could have been # replaced but hasn't (eg partially applied squashed # migration) _reraise_missing_dependency(migration, parent, e) for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] == key[0]: # Internal dependencies already added. continue parent = self.check_key(parent, key[0]) if parent is not None: try: self.graph.add_dependency(migration, key, parent) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "parent" is not in there. _reraise_missing_dependency(migration, parent, e) for child in migration.run_before: child = self.check_key(child, key[0]) if child is not None: try: self.graph.add_dependency(migration, child, key) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "child" is not in there. _reraise_missing_dependency(migration, child, e)
def test_loading_squashed_ref_squashed(self): "Tests loading a squashed migration with a new migration referencing it" r""" The sample migrations are structured like this: app_1 1 --> 2 ---------------------*--> 3 *--> 4 \ / / *-------------------*----/--> 2_sq_3 --* \ / / =============== \ ============= / == / ====================== app_2 *--> 1_sq_2 --* / \ / *--> 1 --> 2 --* Where 2_sq_3 is a replacing migration for 2 and 3 in app_1, as 1_sq_2 is a replacing migration for 1 and 2 in app_2. """ loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) # Load with nothing applied: both migrations squashed. loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations.keys() expected_plan = { ('app1', '1_auto'), ('app2', '1_squashed_2'), ('app1', '2_squashed_3'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan) # Load with nothing applied and migrate to a replaced migration. # Not possible if loader.replace_migrations is True (default). loader.build_graph() msg = "Node ('app1', '3_auto') not a valid node" with self.assertRaisesMessage(NodeNotFoundError, msg): loader.graph.forwards_plan(('app1', '3_auto')) # Possible if loader.replace_migrations is False. loader.replace_migrations = False loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '3_auto'))) plan = plan - loader.applied_migrations.keys() expected_plan = { ('app1', '1_auto'), ('app2', '1_auto'), ('app2', '2_auto'), ('app1', '2_auto'), ('app1', '3_auto'), } self.assertEqual(plan, expected_plan) loader.replace_migrations = True # Fake-apply a few from app1: unsquashes migration in app1. self.record_applied(recorder, 'app1', '1_auto') self.record_applied(recorder, 'app1', '2_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations.keys() expected_plan = { ('app2', '1_squashed_2'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan) # Fake-apply one from app2: unsquashes migration in app2 too. self.record_applied(recorder, 'app2', '1_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations.keys() expected_plan = { ('app2', '2_auto'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan)
def tearDown(self): # Reset applied-migrations state. for db in self.databases: recorder = MigrationRecorder(connections[db]) recorder.migration_qs.filter(app='migrations').delete()
# -*- coding: utf-8 -*- from django.db import models, migrations, DEFAULT_DB_ALIAS, connections from django.db.migrations.recorder import MigrationRecorder import django.db.models.deletion from django.conf import settings connection = connections[DEFAULT_DB_ALIAS] recorder = MigrationRecorder(connection) linaro_django_xmlrpc_applied = False lava_scheduler_app_applied = False for app, name in recorder.applied_migrations(): if app == "linaro_django_xmlrpc" and name == "0001_initial": linaro_django_xmlrpc_applied = True if app == "lava_scheduler_app" and name == "0001_initial": lava_scheduler_app_applied = True if not linaro_django_xmlrpc_applied and lava_scheduler_app_applied: recorder.record_applied("linaro_django_xmlrpc", "0001_initial") class Migration(migrations.Migration): dependencies = [ ("auth", "0001_initial"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("linaro_django_xmlrpc", "__first__"), ("dashboard_app", "__first__"), ] operations = [ migrations.CreateModel( name="DefaultDeviceOwner",
def tearDown(self): super(TestTenantSchemaOperations, self).tearDown() # Make sure all migrations are considered unapplied. MigrationRecorder(connection).flush()
def build_graph(self, ignore_unmigrated=False): """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do a first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Carry out replacements if we can - that is, if all replaced migrations # are either unapplied or missing. for key, migration in replacing.items(): # Ensure this replacement migration is not in applied_migrations self.applied_migrations.discard(key) # Do the check. We can replace if all our replace targets are # applied, or if all of them are unapplied. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] can_replace = all(applied_statuses) or (not any(applied_statuses)) if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): if child_key in migration.replaces: continue normal[child_key].dependencies.remove(replaced) normal[child_key].dependencies.append(key) normal[key] = migration # Mark the replacement as applied if all its replaced ones are if all(applied_statuses): self.applied_migrations.add(key) # Finally, make a graph and load everything into it self.graph = MigrationGraph() for key, migration in normal.items(): self.graph.add_node(key, migration) for key, migration in normal.items(): for parent in migration.dependencies: # Special-case __first__, which means "the first migration" for # migrated apps, and is ignored for unmigrated apps. It allows # makemigrations to declare dependencies on apps before they # even have migrations. if parent[1] == "__first__" and parent not in self.graph: if parent[0] in self.unmigrated_apps: if ignore_unmigrated: migration.dependencies.remove(parent) parent = None else: # This app isn't migrated, but something depends on it. # We'll add a fake initial migration for it into the # graph. app_config = apps.get_app_config(parent[0]) ops = [] for model in app_config.get_models(): model_state = ModelState.from_model(model) ops.append( operations.CreateModel( name=model_state.name, fields=model_state.fields, options=model_state.options, bases=model_state.bases, )) new_migration = type( "FakeInitialMigration", (Migration, ), {"operations": ops}, )(parent[1], parent[0]) self.graph.add_node(parent, new_migration) self.applied_migrations.add(parent) elif parent[0] in self.migrated_apps: parent = list(self.graph.root_nodes(parent[0]))[0] else: raise ValueError("Dependency on unknown app %s" % parent[0]) if parent is not None: self.graph.add_dependency(key, parent)
def tearDown(self): # Reset applied-migrations state. for db in connections: MigrationRecorder(connections[db])
def fake_initial_users_migrations(): recorder = MigrationRecorder(connection) recorder.record_applied("users", "0001_initial")