def check_consistent_history(self, connection): """ Raise InconsistentMigrationHistory if any applied migrations have unapplied dependencies. """ recorder = MigrationRecorder(connection) try: applied = recorder.applied_migrations() except MigrationSchemaMissing: # Skip check if the django_migrations table is missing and can't be # created. return for migration in applied: # If the migration is unknown, skip it. if migration not in self.graph.nodes: continue for parent in self.graph.node_map[migration].parents: if parent not in applied: # Skip unapplied squashed migrations that have all of their # `replaces` applied. if parent in self.replacements: if all(m in applied for m in self.replacements[parent].replaces): continue raise InconsistentMigrationHistory( "Migration {}.{} is applied before its dependency " "{}.{} on database '{}'.".format( migration[0], migration[1], parent[0], parent[1], connection.alias, ) )
def build_graph(self): """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do a first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Carry out replacements if we can - that is, if all replaced migrations # are either unapplied or missing. for key, migration in replacing.items(): # Ensure this replacement migration is not in applied_migrations self.applied_migrations.discard(key) # Do the check. We can replace if all our replace targets are # applied, or if all of them are unapplied. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] can_replace = all(applied_statuses) or (not any(applied_statuses)) if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): if child_key in migration.replaces: continue normal[child_key].dependencies.remove(replaced) normal[child_key].dependencies.append(key) normal[key] = migration # Mark the replacement as applied if all its replaced ones are if all(applied_statuses): self.applied_migrations.add(key) # Finally, make a graph and load everything into it self.graph = MigrationGraph() for key, migration in normal.items(): self.graph.add_node(key, migration) for key, migration in normal.items(): for parent in migration.dependencies: self.graph.add_dependency(key, parent)
def test_apply(self): """ Tests marking migrations as applied/unapplied. """ recorder = MigrationRecorder(connection) self.assertEqual( set((x, y) for (x, y) in recorder.applied_migrations() if x == "myapp"), set(), ) recorder.record_applied("myapp", "0432_ponies") self.assertEqual( set((x, y) for (x, y) in recorder.applied_migrations() if x == "myapp"), {("myapp", "0432_ponies")}, ) # That should not affect records of another database recorder_other = MigrationRecorder(connections['other']) self.assertEqual( set((x, y) for (x, y) in recorder_other.applied_migrations() if x == "myapp"), set(), ) recorder.record_unapplied("myapp", "0432_ponies") self.assertEqual( set((x, y) for (x, y) in recorder.applied_migrations() if x == "myapp"), set(), )
def test_check_consistent_history(self): loader = MigrationLoader(connection=None) loader.check_consistent_history(connection) recorder = MigrationRecorder(connection) recorder.record_applied('migrations', '0002_second') msg = "Migration migrations.0002_second is applied before its dependency migrations.0001_initial" with self.assertRaisesMessage(InconsistentMigrationHistory, msg): loader.check_consistent_history(connection)
def test_loading_squashed_ref_squashed(self): "Tests loading a squashed migration with a new migration referencing it" r""" The sample migrations are structured like this: app_1 1 --> 2 ---------------------*--> 3 *--> 4 \ / / *-------------------*----/--> 2_sq_3 --* \ / / =============== \ ============= / == / ====================== app_2 *--> 1_sq_2 --* / \ / *--> 1 --> 2 --* Where 2_sq_3 is a replacing migration for 2 and 3 in app_1, as 1_sq_2 is a replacing migration for 1 and 2 in app_2. """ loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) # Load with nothing applied: both migrations squashed. loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations expected_plan = { ('app1', '1_auto'), ('app2', '1_squashed_2'), ('app1', '2_squashed_3'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan) # Fake-apply a few from app1: unsquashes migration in app1. recorder.record_applied('app1', '1_auto') recorder.record_applied('app1', '2_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations expected_plan = { ('app2', '1_squashed_2'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan) # Fake-apply one from app2: unsquashes migration in app2 too. recorder.record_applied('app2', '1_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations expected_plan = { ('app2', '2_auto'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan)
def graph(self): """ Builds a migration dependency graph using both the disk and database. """ # Make sure we have the disk data if self.disk_migrations is None: self.load_disk() # And the database data if self.applied_migrations is None: recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do a first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Carry out replacements if we can - that is, if all replaced migrations # are either unapplied or missing. for key, migration in replacing.items(): # Do the check can_replace = True for target in migration.replaces: if target in self.applied_migrations: can_replace = False break if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): normal[child_key].dependencies.remove(replaced) normal[child_key].dependencies.append(key) normal[key] = migration # Finally, make a graph and load everything into it graph = MigrationGraph() for key, migration in normal.items(): graph.add_node(key, migration) for key, migration in normal.items(): for parent in migration.dependencies: graph.add_dependency(key, parent) return graph
def test_loading_squashed(self): "Tests loading a squashed migration" migration_loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) # Loading with nothing applied should just give us the one node self.assertEqual(len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]), 1) # However, fake-apply one migration and it should now use the old two recorder.record_applied("migrations", "0001_initial") migration_loader.build_graph() self.assertEqual(len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]), 2) recorder.flush()
def test_migrations_not_applied_on_deferred_sql_failure(self): """Migrations are not recorded if deferred SQL application fails.""" class DeferredSQL: def __str__(self): raise DatabaseError('Failed to apply deferred SQL') class Migration(migrations.Migration): atomic = False def apply(self, project_state, schema_editor, collect_sql=False): schema_editor.deferred_sql.append(DeferredSQL()) executor = MigrationExecutor(connection) with self.assertRaisesMessage(DatabaseError, 'Failed to apply deferred SQL'): executor.apply_migration( ProjectState(), Migration('0001_initial', 'deferred_sql'), ) # The migration isn't recorded as applied since it failed. migration_recorder = MigrationRecorder(connection) self.assertIs( migration_recorder.migration_qs.filter( app='deferred_sql', name='0001_initial', ).exists(), False, )
def test_migrations_applied_and_recorded_atomically(self): """Migrations are applied and recorded atomically.""" class Migration(migrations.Migration): operations = [ migrations.CreateModel('model', [ ('id', models.AutoField(primary_key=True)), ]), ] executor = MigrationExecutor(connection) with mock.patch( 'django.db.migrations.executor.MigrationExecutor.record_migration' ) as record_migration: record_migration.side_effect = RuntimeError( 'Recording migration failed.') with self.assertRaisesMessage(RuntimeError, 'Recording migration failed.'): executor.apply_migration( ProjectState(), Migration('0001_initial', 'record_migration'), ) executor.migrate([('migrations', '0001_initial')]) # The migration isn't recorded as applied since it failed. migration_recorder = MigrationRecorder(connection) self.assertIs( migration_recorder.migration_qs.filter( app='record_migration', name='0001_initial', ).exists(), False, ) self.assertTableNotExists('record_migration_model')
def test_loading_squashed_complex_multi_apps_partially_applied(self): loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) recorder.record_applied('app1', '1_auto') recorder.record_applied('app1', '2_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations expected_plan = { ('app2', '1_squashed_2'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan)
def _get_migrations(included_apps): """ Get migrations for included apps. """ migration_objects = [] for app_config in apps.get_app_configs(): if app_config.name not in included_apps: continue app_label = app_config.label module_name, _ = MigrationLoader.migrations_module(app_label) if module_name is None: continue try: module = import_module(module_name) except ImportError: continue directory = os.path.dirname(module.__file__) for name in os.listdir(directory): if name.endswith(".py"): import_name = name.rsplit(".", 1)[0] if import_name[0] not in "_.~": migration_objects.append( MigrationRecorder.Migration(app=app_label, name=import_name)) return migration_objects
def test_migrate_record_replaced(self): """ Running a single squashed migration should record all of the original replaced migrations as run. """ recorder = MigrationRecorder(connection) out = six.StringIO() call_command("migrate", "migrations", verbosity=0) call_command("showmigrations", "migrations", stdout=out, no_color=True) self.assertEqual("migrations\n" " [x] 0001_squashed_0002 (2 squashed migrations)\n", out.getvalue().lower()) applied_migrations = recorder.applied_migrations() self.assertIn(("migrations", "0001_initial"), applied_migrations) self.assertIn(("migrations", "0002_second"), applied_migrations) self.assertIn(("migrations", "0001_squashed_0002"), applied_migrations) # Rollback changes call_command("migrate", "migrations", "zero", verbosity=0)
def get_applied_migrations(self): return { name for (app_label, name) in MigrationRecorder(connection).applied_migrations() if app_label == "tests" }
def test_loading_squashed_complex_multi_apps_partially_applied(self): loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) recorder.record_applied('app1', '1_auto') recorder.record_applied('app1', '2_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations expected_plan = { ('app1', '4_auto'), ('app1', '3_auto'), ('app2', '1_squashed_2'), } self.assertEqual(plan, expected_plan)
def _migrate_schema(self, connection, tenant): created = False if not schema_exists(connection, tenant.schema_name): connection.cursor().execute('CREATE SCHEMA "%s"' % tenant.schema_name) created = True MigrationRecorder(connection).ensure_schema() return created
def test_loading_squashed(self): "Tests loading a squashed migration" migration_loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) # Loading with nothing applied should just give us the one node self.assertEqual( len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]), 1, ) # However, fake-apply one migration and it should now use the old two recorder.record_applied("migrations", "0001_initial") migration_loader.build_graph() self.assertEqual( len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]), 2, )
def get_applied_migrations(version, connection): """ Return the list of applied migrations for the given version. Reuse django migration table. """ recorder = MigrationRecorder(connection) return list(recorder.migration_qs.filter(app=version).values_list( 'name', flat=True))
def test_loading_squashed_ref_squashed(self): "Tests loading a squashed migration with a new migration referencing it" r""" The sample migrations are structured like this: app_1 1 --> 2 ---------------------*--> 3 *--> 4 \ / / *-------------------*----/--> 2_sq_3 --* \ / / =============== \ ============= / == / ====================== app_2 *--> 1_sq_2 --* / \ / *--> 1 --> 2 --* Where 2_sq_3 is a replacing migration for 2 and 3 in app_1, as 1_sq_2 is a replacing migration for 1 and 2 in app_2. """ loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) # Load with nothing applied: both migrations squashed. loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations.keys() expected_plan = { ('app1', '1_auto'), ('app2', '1_squashed_2'), ('app1', '2_squashed_3'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan) # Fake-apply a few from app1: unsquashes migration in app1. self.record_applied(recorder, 'app1', '1_auto') self.record_applied(recorder, 'app1', '2_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations.keys() expected_plan = { ('app2', '1_squashed_2'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan) # Fake-apply one from app2: unsquashes migration in app2 too. self.record_applied(recorder, 'app2', '1_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations.keys() expected_plan = { ('app2', '2_auto'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan)
def check_consistent_history(self, connection): """ Raise InconsistentMigrationHistory if any applied migrations have unapplied dependencies. """ recorder = MigrationRecorder(connection) applied = recorder.applied_migrations() for migration in applied: # If the migration is unknown, skip it. if migration not in self.graph.nodes: continue for parent in self.graph.node_map[migration].parents: if parent not in applied: raise InconsistentMigrationHistory( "Migration {}.{} is applied before its dependency {}.{}".format( migration[0], migration[1], parent[0], parent[1], ) )
def test_apply_all_replaced_marks_replacement_as_applied(self): """ Applying all replaced migrations marks replacement as applied (#24628). """ recorder = MigrationRecorder(connection) # Place the database in a state where the replaced migrations are # partially applied: 0001 is applied, 0002 is not. recorder.record_applied("migrations", "0001_initial") executor = MigrationExecutor(connection) # Use fake because we don't actually have the first migration # applied, so the second will fail. And there's no need to actually # create/modify tables here, we're just testing the # MigrationRecord, which works the same with or without fake. executor.migrate([("migrations", "0002_second")], fake=True) # Because we've now applied 0001 and 0002 both, their squashed # replacement should be marked as applied. self.assertIn(("migrations", "0001_squashed_0002"), recorder.applied_migrations())
def test_check_consistent_history_squashed(self): """ MigrationLoader.check_consistent_history() should ignore unapplied squashed migrations that have all of their `replaces` applied. """ loader = MigrationLoader(connection=None) recorder = MigrationRecorder(connection) recorder.record_applied('migrations', '0001_initial') recorder.record_applied('migrations', '0002_second') loader.check_consistent_history(connection) recorder.record_applied('migrations', '0003_third') loader.check_consistent_history(connection)
def test_migrate_record_replaced(self): """ Running a single squashed migration should record all of the original replaced migrations as run. """ recorder = MigrationRecorder(connection) out = six.StringIO() call_command("migrate", "migrations", verbosity=0) call_command("showmigrations", "migrations", stdout=out, no_color=True) self.assertEqual( 'migrations\n' ' [x] 0001_squashed_0002 (2 squashed migrations)\n', out.getvalue().lower()) applied_migrations = recorder.applied_migrations() self.assertIn(("migrations", "0001_initial"), applied_migrations) self.assertIn(("migrations", "0002_second"), applied_migrations) self.assertIn(("migrations", "0001_squashed_0002"), applied_migrations) # Rollback changes call_command("migrate", "migrations", "zero", verbosity=0)
def test_check_consistent_history(self): loader = MigrationLoader(connection=None) loader.check_consistent_history(connection) recorder = MigrationRecorder(connection) self.record_applied(recorder, "migrations", "0002_second") msg = ( "Migration migrations.0002_second is applied before its dependency " "migrations.0001_initial on database 'default'.") with self.assertRaisesMessage(InconsistentMigrationHistory, msg): loader.check_consistent_history(connection)
def app_has_unapplied_migrations(app_name): # Note: We cannot configure Django globally, because some config files # might not exist yet. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "initat.cluster.settings") connection, apps, MigrationRecorder = setup_django() recorder = MigrationRecorder(connection) applied_migrations = { migration_name for app_name_, migration_name in recorder.applied_migrations() if app_name_ == app_name } migration_directory = os.path.join(apps.get_app_config(app_name).path, "migrations") migrations_on_disk = set( fnmatch.filter(os.listdir(migration_directory), "*.py") ) migrations_on_disk = {os.path.splitext(i)[0] for i in migrations_on_disk} migrations_on_disk.remove("__init__") return len(migrations_on_disk - applied_migrations) > 0
def check_consistent_migration_file(self, connection): """ Raise InconsistentMigrationFileHistory if a migration file is deleted. """ recorder = MigrationRecorder(connection) applied = recorder.applied_migrations() # If the migration is unknown, record that as removed file deleted_migrations = {} for migration in applied: if migration not in self.graph.nodes: deleted_migrations.setdefault(migration[0], []).append(migration[1]) error_messages = [] for app_label in deleted_migrations: error_messages.append(" App '%s':\n" % app_label) for file_name in deleted_migrations[app_label]: error_messages.append(" \t'%s'\n" % file_name) if error_messages: raise InconsistentMigrationFileHistory( "WARNING: Inconsistent migrations\n%s" % "".join(error_messages))
def handle(self, *app_labels, **options): Migration = MigrationRecorder(None).Migration for app_label in app_labels: count = Migration.objects.filter(app=app_label).count() ans = input('Are you sure you want to delete all (%s) migrations ' 'for %s? [Y/n] ' % (count, app_label)) if ans == 'Y': Migration.objects.filter(app=app_label).delete() print('Done.') else: print('No action.')
def test_migrations_applied_and_recorded_atomically(self): """Migrations are applied and recorded atomically.""" executor = MigrationExecutor(connection) with mock.patch('django.db.migrations.executor.MigrationExecutor.record_migration') as record_migration: record_migration.side_effect = RuntimeError('Recording migration failed.') with self.assertRaisesMessage(RuntimeError, 'Recording migration failed.'): executor.migrate([('migrations', '0001_initial')]) # The migration isn't recorded as applied since it failed. migration_recorder = MigrationRecorder(connection) self.assertFalse(migration_recorder.migration_qs.filter(app='migrations', name='0001_initial').exists()) self.assertTableNotExists('migrations_author')
def test_apply_all_replaced_marks_replacement_as_applied(self): """ Applying all replaced migrations marks replacement as applied (#24628). """ recorder = MigrationRecorder(connection) # Place the database in a state where the replaced migrations are # partially applied: 0001 is applied, 0002 is not. recorder.record_applied("migrations", "0001_initial") executor = MigrationExecutor(connection) # Use fake because we don't actually have the first migration # applied, so the second will fail. And there's no need to actually # create/modify tables here, we're just testing the # MigrationRecord, which works the same with or without fake. executor.migrate([("migrations", "0002_second")], fake=True) # Because we've now applied 0001 and 0002 both, their squashed # replacement should be marked as applied. self.assertIn( ("migrations", "0001_squashed_0002"), recorder.applied_migrations(), )
def _populate_migrations(self, connection, included_apps): MigrationLoader.migrations_module = _original_migrations_module migration_objects = migrate_helpers.get_migrations(included_apps) MigrationRecorder(connection).migration_qs.bulk_create( migration_objects) # Reset sequences statements = connection.ops.sequence_reset_sql( no_style(), (MigrationRecorder.Migration, )) with connection.cursor() as cursor: for line in statements: cursor.execute(line)
def handle(self, *args, **options): # Get the database we're operating from db = options['database'] connection = connections[db] # Hook for backends needing any database preparation connection.prepare_database() # Detect mid-project installation recorder = MigrationRecorder(connection) if (_has_table(recorder) and recorder.migration_qs.filter(app='auth').exists() and not recorder.migration_qs.filter( app='user_unique_email').exists()): # Auto fake initial migration migrations = [ recorder.Migration(app='user_unique_email', name='0001_initial') ] + [ recorder.Migration(app=m.app, name=m.name) for m in recorder.migration_qs.iterator() ] recorder.flush() recorder.migration_qs.bulk_create(migrations) # Go on with normal migrate command return super().handle(*args, **options)
def handle(self, *args, **options): # check if auth is not finish migrations db = options['database'] connection = connections[db] recorder = MigrationRecorder(connection) applied = recorder.applied_migrations() if applied: is_auth_finish = False target_init_path = os.path.join(settings.BASE_DIR, 'blueapps', 'account', 'migrations', '__init__.py') target_init_pyc_path = os.path.join(settings.BASE_DIR, 'blueapps', 'account', 'migrations', '__init__.pyc') for migration in applied: if migration[0] == 'auth' and migration[1].startswith('0007'): is_auth_finish = True if settings.RUN_VER == 'open' and not is_auth_finish: sys.stdout.write('remove init for auth is not finish.\n') try: os.remove(target_init_path) os.remove(target_init_pyc_path) except FileNotFoundError: pass sys.stdout.write('ready to execute the true migrate\n') super(Command, self).handle(*args, **options) if not is_auth_finish: open(target_init_path, 'w') sys.stdout.write( 'ready to execute the true migrate again\n') super(Command, self).handle(*args, **options)
def get_latest_applied_migrations_qs( connection_obj=None, ) -> "QuerySet[MigrationRecorder.Migration]": """Return latest applied migration in all django apps in project""" if connection_obj is None: connection_obj = connection recorder = MigrationRecorder(connection_obj) migration_qs = recorder.migration_qs.exclude(app=SELF_NAME) lastest_ids = (migration_qs.values("app").annotate( latest_id=Max("id")).values_list( "latest_id", flat=True, )) return migration_qs.filter(id__in=lastest_ids)
def test_record_applied_migrations(self): """Testing record_applied_migrations""" connection = connections[DEFAULT_DB_ALIAS] # Ideally we'd do an assertNumQueries(2), but MigrationRecorder doesn't # cache state and performs repeated queries for the same list of # installed table names, followed by new transactions. That might # differ depending on the type of database being used. migrations = MigrationList() migrations.add_migration_info(app_label='tests', name='0001_initial') migrations.add_migration_info(app_label='tests', name='0002_stuff') record_applied_migrations(connection=connection, migrations=migrations) recorder = MigrationRecorder(connection) applied_migrations = recorder.applied_migrations() self.assertIn(('tests', '0001_initial'), applied_migrations) self.assertIn(('tests', '0002_stuff'), applied_migrations)
def test_loading_squashed_erroneous(self): "Tests loading a complex but erroneous set of squashed migrations" loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) def num_nodes(): plan = set(loader.graph.forwards_plan(("migrations", "7_auto"))) return len(plan - loader.applied_migrations.keys()) # Empty database: use squashed migration loader.build_graph() self.assertEqual(num_nodes(), 5) # Starting at 1 or 2 should use the squashed migration too self.record_applied(recorder, "migrations", "1_auto") loader.build_graph() self.assertEqual(num_nodes(), 4) self.record_applied(recorder, "migrations", "2_auto") loader.build_graph() self.assertEqual(num_nodes(), 3) # However, starting at 3 or 4, nonexistent migrations would be needed. msg = ( "Migration migrations.6_auto depends on nonexistent node " "('migrations', '5_auto'). Django tried to replace migration " "migrations.5_auto with any of [migrations.3_squashed_5] but wasn't able " "to because some of the replaced migrations are already applied.") self.record_applied(recorder, "migrations", "3_auto") with self.assertRaisesMessage(NodeNotFoundError, msg): loader.build_graph() self.record_applied(recorder, "migrations", "4_auto") with self.assertRaisesMessage(NodeNotFoundError, msg): loader.build_graph() # Starting at 5 to 7 we are passed the squashed migrations self.record_applied(recorder, "migrations", "5_auto") loader.build_graph() self.assertEqual(num_nodes(), 2) self.record_applied(recorder, "migrations", "6_auto") loader.build_graph() self.assertEqual(num_nodes(), 1) self.record_applied(recorder, "migrations", "7_auto") loader.build_graph() self.assertEqual(num_nodes(), 0)
def test_apply(self): """ Tests marking migrations as applied/unapplied. """ recorder = MigrationRecorder(connection) self.assertEqual( recorder.applied_migrations(), set(), ) recorder.record_applied("myapp", "0432_ponies") self.assertEqual( recorder.applied_migrations(), set([("myapp", "0432_ponies")]), ) recorder.record_unapplied("myapp", "0432_ponies") self.assertEqual( recorder.applied_migrations(), set(), )
def test_unrecord_applied_migrations(self): """Testing unrecord_applied_migrations""" connection = connections[DEFAULT_DB_ALIAS] migrations = MigrationList() migrations.add_migration_info(app_label='tests', name='0001_initial') migrations.add_migration_info(app_label='tests', name='0002_stuff') record_applied_migrations(connection=connection, migrations=migrations) unrecord_applied_migrations(connection=connection, app_label='tests', migration_names=['0001_initial', '0002_stuff']) recorder = MigrationRecorder(connection) applied_migrations = recorder.applied_migrations() self.assertNotIn(('tests', '0001_initial'), applied_migrations) self.assertNotIn(('tests', '0002_stuff'), applied_migrations)
def _dump_meta(archive): with Command._write_to_archive(archive, 'meta.json') as fileobj: dump( { 'version': __version__, 'migrations': dict( MigrationRecorder( connection).applied_migrations().keys()), }, fileobj, indent=2, )
def record_applied_migrations(connection, migrations): """Record a list of applied migrations to the database. This can only be called when on Django 1.7 or higher. Args: connection (django.db.backends.base.BaseDatabaseWrapper): The connection used to record applied migrations. migrations (MigrationList): The list of migration targets to record as applied. """ assert supports_migrations, \ 'This cannot be called on Django 1.6 or earlier.' recorder = MigrationRecorder(connection) recorder.ensure_schema() recorder.migration_qs.bulk_create( recorder.Migration(app=info['app_label'], name=info['name']) for info in migrations )
def change_state_mock(self, snapshot): # pylint: disable=unused-argument # unfortunately no other way that we can chane state in tests # without all migrations files and whole django machinery # so for this test we will mock this state recorder = MigrationRecorder(connection=connection) Migration = recorder.Migration migration_state = snapshot.migrations.values_list("app", "name") with transaction.atomic(): Migration.objects.all().delete() for app, name in migration_state: m = Migration( app=app, name=name, ) m.save()
def test_loading_squashed_complex_multi_apps_partially_applied(self): loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.record_applied(recorder, "app1", "1_auto") self.record_applied(recorder, "app1", "2_auto") loader.build_graph() plan = set(loader.graph.forwards_plan(("app1", "4_auto"))) plan = plan - loader.applied_migrations.keys() expected_plan = { ("app2", "1_squashed_2"), ("app1", "3_auto"), ("app1", "4_auto"), } self.assertEqual(plan, expected_plan)
def test_migrate_record_squashed(self): """ Running migrate for a squashed migration should record as run if all of the replaced migrations have been run (#25231). """ recorder = MigrationRecorder(connection) recorder.record_applied("migrations", "0001_initial") recorder.record_applied("migrations", "0002_second") out = six.StringIO() call_command("migrate", "migrations", verbosity=0) call_command("showmigrations", "migrations", stdout=out, no_color=True) self.assertEqual("migrations\n" " [x] 0001_squashed_0002 (2 squashed migrations)\n", out.getvalue().lower()) self.assertIn(("migrations", "0001_squashed_0002"), recorder.applied_migrations())
def test_migrate_marks_replacement_applied_even_if_it_did_nothing(self): """ A new squash migration will be marked as applied even if all its replaced migrations were previously already applied (#24628). """ recorder = MigrationRecorder(connection) # Record all replaced migrations as applied recorder.record_applied("migrations", "0001_initial") recorder.record_applied("migrations", "0002_second") executor = MigrationExecutor(connection) executor.migrate([("migrations", "0001_squashed_0002")]) # Because 0001 and 0002 are both applied, even though this migrate run # didn't apply anything new, their squashed replacement should be # marked as applied. self.assertIn(("migrations", "0001_squashed_0002"), recorder.applied_migrations())
def build_graph(self): """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data if self.connection is None: self.applied_migrations = set() else: recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do a first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Remember the possible replacements to generate more meaningful error # messages reverse_replacements = {} for key, migration in replacing.items(): for replaced in migration.replaces: reverse_replacements.setdefault(replaced, set()).add(key) # Carry out replacements if we can - that is, if all replaced migrations # are either unapplied or missing. for key, migration in replacing.items(): # Ensure this replacement migration is not in applied_migrations self.applied_migrations.discard(key) # Do the check. We can replace if all our replace targets are # applied, or if all of them are unapplied. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] can_replace = all(applied_statuses) or (not any(applied_statuses)) if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): if child_key in migration.replaces: continue # child_key may appear in a replacement if child_key in reverse_replacements: for replaced_child_key in reverse_replacements[child_key]: if replaced in replacing[replaced_child_key].dependencies: replacing[replaced_child_key].dependencies.remove(replaced) replacing[replaced_child_key].dependencies.append(key) else: normal[child_key].dependencies.remove(replaced) normal[child_key].dependencies.append(key) normal[key] = migration # Mark the replacement as applied if all its replaced ones are if all(applied_statuses): self.applied_migrations.add(key) # Finally, make a graph and load everything into it self.graph = MigrationGraph() for key, migration in normal.items(): self.graph.add_node(key, migration) def _reraise_missing_dependency(migration, missing, exc): """ Checks if ``missing`` could have been replaced by any squash migration but wasn't because the the squash migration was partially applied before. In that case raise a more understandable exception. #23556 """ if missing in reverse_replacements: candidates = reverse_replacements.get(missing, set()) is_replaced = any(candidate in self.graph.nodes for candidate in candidates) if not is_replaced: tries = ', '.join('%s.%s' % c for c in candidates) exc_value = NodeNotFoundError( "Migration {0} depends on nonexistent node ('{1}', '{2}'). " "Django tried to replace migration {1}.{2} with any of [{3}] " "but wasn't able to because some of the replaced migrations " "are already applied.".format( migration, missing[0], missing[1], tries ), missing) exc_value.__cause__ = exc six.reraise(NodeNotFoundError, exc_value, sys.exc_info()[2]) raise exc # Add all internal dependencies first to ensure __first__ dependencies # find the correct root node. for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] != key[0] or parent[1] == '__first__': # Ignore __first__ references to the same app (#22325) continue try: self.graph.add_dependency(migration, key, parent) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "parent" is not in there. To make the raised exception # more understandable we check if parent could have been # replaced but hasn't (eg partially applied squashed # migration) _reraise_missing_dependency(migration, parent, e) for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] == key[0]: # Internal dependencies already added. continue parent = self.check_key(parent, key[0]) if parent is not None: try: self.graph.add_dependency(migration, key, parent) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "parent" is not in there. _reraise_missing_dependency(migration, parent, e) for child in migration.run_before: child = self.check_key(child, key[0]) if child is not None: try: self.graph.add_dependency(migration, child, key) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "child" is not in there. _reraise_missing_dependency(migration, child, e)
def build_graph(self): """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data if self.connection is None: self.applied_migrations = set() else: recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # To start, populate the migration graph with nodes for ALL migrations # and their dependencies. Also make note of replacing migrations at this step. self.graph = MigrationGraph() self.replacements = {} for key, migration in self.disk_migrations.items(): self.graph.add_node(key, migration) # Internal (aka same-app) dependencies. self.add_internal_dependencies(key, migration) # Replacing migrations. if migration.replaces: self.replacements[key] = migration # Add external dependencies now that the internal ones have been resolved. for key, migration in self.disk_migrations.items(): self.add_external_dependencies(key, migration) # Carry out replacements where possible. for key, migration in self.replacements.items(): # Get applied status of each of this migration's replacement targets. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] # Ensure the replacing migration is only marked as applied if all of # its replacement targets are. if all(applied_statuses): self.applied_migrations.add(key) else: self.applied_migrations.discard(key) # A replacing migration can be used if either all or none of its # replacement targets have been applied. if all(applied_statuses) or (not any(applied_statuses)): self.graph.remove_replaced_nodes(key, migration.replaces) else: # This replacing migration cannot be used because it is partially applied. # Remove it from the graph and remap dependencies to it (#25945). self.graph.remove_replacement_node(key, migration.replaces) # Ensure the graph is consistent. try: self.graph.validate_consistency() except NodeNotFoundError as exc: # Check if the missing node could have been replaced by any squash # migration but wasn't because the squash migration was partially # applied before. In that case raise a more understandable exception # (#23556). # Get reverse replacements. reverse_replacements = {} for key, migration in self.replacements.items(): for replaced in migration.replaces: reverse_replacements.setdefault(replaced, set()).add(key) # Try to reraise exception with more detail. if exc.node in reverse_replacements: candidates = reverse_replacements.get(exc.node, set()) is_replaced = any(candidate in self.graph.nodes for candidate in candidates) if not is_replaced: tries = ', '.join('%s.%s' % c for c in candidates) exc_value = NodeNotFoundError( "Migration {0} depends on nonexistent node ('{1}', '{2}'). " "Django tried to replace migration {1}.{2} with any of [{3}] " "but wasn't able to because some of the replaced migrations " "are already applied.".format( exc.origin, exc.node[0], exc.node[1], tries ), exc.node ) exc_value.__cause__ = exc if not hasattr(exc, '__traceback__'): exc.__traceback__ = sys.exc_info()[2] six.reraise(NodeNotFoundError, exc_value, sys.exc_info()[2]) raise exc
def build_graph(self): """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do a first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Carry out replacements if we can - that is, if all replaced migrations # are either unapplied or missing. for key, migration in replacing.items(): # Ensure this replacement migration is not in applied_migrations self.applied_migrations.discard(key) # Do the check. We can replace if all our replace targets are # applied, or if all of them are unapplied. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces] can_replace = all(applied_statuses) or (not any(applied_statuses)) if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): if child_key in migration.replaces: continue normal[child_key].dependencies.remove(replaced) normal[child_key].dependencies.append(key) normal[key] = migration # Mark the replacement as applied if all its replaced ones are if all(applied_statuses): self.applied_migrations.add(key) # Finally, make a graph and load everything into it self.graph = MigrationGraph() for key, migration in normal.items(): self.graph.add_node(key, migration) for key, migration in normal.items(): for parent in migration.dependencies: # Special-case __first__, which means "the first migration" for # migrated apps, and is ignored for unmigrated apps. It allows # makemigrations to declare dependencies on apps before they # even have migrations. if parent[1] == "__first__" and parent not in self.graph: if parent[0] in self.unmigrated_apps: # This app isn't migrated, but something depends on it. # The models will get auto-added into the state, though # so we're fine. continue elif parent[0] in self.migrated_apps: parent = list(self.graph.root_nodes(parent[0]))[0] else: raise ValueError("Dependency on unknown app %s" % parent[0]) if parent is not None: self.graph.add_dependency(key, parent)
from __future__ import unicode_literals
def build_graph(self): # pragma: no cover """ Builds a migration dependency graph using both the disk and database. You'll need to rebuild the graph if you apply migrations. This isn't usually a problem as generally migration stuff runs in a one-shot process. """ # Load disk data self.load_disk() # Load database data if self.connection is None: self.applied_migrations = set() else: recorder = MigrationRecorder(self.connection) self.applied_migrations = recorder.applied_migrations() # Do first pass to separate out replacing and non-replacing migrations normal = {} replacing = {} for key, migration in self.disk_migrations.items(): if migration.replaces: replacing[key] = migration else: normal[key] = migration # Calculate reverse dependencies - i.e., for each migration, # what depends on it? # This is just for dependency re-pointing when applying replacements, # so we ignore run_before here. reverse_dependencies = {} for key, migration in normal.items(): for parent in migration.dependencies: reverse_dependencies.setdefault(parent, set()).add(key) # Remember the possible replacements to generate more meaningful error # messages reverse_replacements = {} for key, migration in replacing.items(): for replaced in migration.replaces: reverse_replacements.setdefault(replaced, set()).add(key) # Carry out replacements if we can - that is, if all replaced migration # are either unapplied or missing. for key, migration in replacing.items(): # Ensure this replacement migration is not in applied_migrations self.applied_migrations.discard(key) # Do the check. We can replace if all our replace targets are # applied, or if all of them are unapplied. applied = self.applied_migrations replaces = migration.replaces applied_statuses = [(target in applied) for target in replaces] can_replace = all(applied_statuses) or (not any(applied_statuses)) if not can_replace: continue # Alright, time to replace. Step through the replaced migrations # and remove, repointing dependencies if needs be. for replaced in migration.replaces: if replaced in normal: # We don't care if the replaced migration doesn't exist; # the usage pattern here is to delete things after a while. del normal[replaced] for child_key in reverse_dependencies.get(replaced, set()): if child_key in migration.replaces: continue # List of migrations whose dependency on `replaced` needs # to be updated to a dependency on `key`. to_update = [] # Child key may itself be replaced, in which case it might # not be in `normal` anymore (depending on whether we've # processed its replacement yet). If it's present, we go # ahead and update it; it may be deleted later on if it is # replaced, but there's no harm in updating it regardless. if child_key in normal: to_update.append(normal[child_key]) # If the child key is replaced, we update its replacement's # dependencies too, if necessary. (We don't know if this # replacement will actually take effect or not, but either # way it's OK to update the replacing migration). if child_key in reverse_replacements: for replaces_ck in reverse_replacements[child_key]: if replaced in replacing[replaces_ck].dependencies: to_update.append(replacing[replaces_ck]) # Actually perform the dependency update on all migrations # that require it. for migration_needing_update in to_update: migration_needing_update.dependencies.remove(replaced) migration_needing_update.dependencies.append(key) normal[key] = migration # Mark the replacement as applied if all its replaced ones are if all(applied_statuses): self.applied_migrations.add(key) # Store the replacement migrations for later checks self.replacements = replacing # Finally, make a graph and load everything into it self.graph = MigrationGraph() for key, migration in normal.items(): self.graph.add_node(key, migration) def _reraise_missing_dependency(migration, missing, exc): """ Checks if ``missing`` could have been replaced by any squash migration but wasn't because the the squash migration was partially applied before. In that case raise a more understandable exception. #23556 """ if missing in reverse_replacements: candidates = reverse_replacements.get(missing, set()) nodes = self.graph.nodees is_replaced = \ any(candidate in nodes for candidate in candidates) if not is_replaced: tries = ', '.join('%s.%s' % c for c in candidates) exc_value = NodeNotFoundError( "Migration {0} depends on nonexistent node " "('{1}', '{2}'). Django tried to replace migration " "{1}.{2} with any of [{3}] but wasn't able to because " "some of the replaced migrations are already " "applied.".format( migration, missing[0], missing[1], tries ), missing) exc_value.__cause__ = exc six.reraise( NodeNotFoundError, exc_value, sys.exc_info()[2]) raise exc # Add all internal dependencies first to ensure __first__ dependencies # find the correct root node. for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] != key[0] or parent[1] == '__first__': # Ignore __first__ references to the same app (#22325) continue try: self.graph.add_dependency(migration, key, parent) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "parent" is not in there. To make the raised exception # more understandable we check if parent could have been # replaced but hasn't (eg partially applied squashed # migration) _reraise_missing_dependency(migration, parent, e) for key, migration in normal.items(): for parent in migration.dependencies: if parent[0] == key[0]: # Internal dependencies already added. continue parent = self.check_key(parent, key[0]) if parent is not None: try: self.graph.add_dependency(migration, key, parent) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "parent" is not in there. _reraise_missing_dependency(migration, parent, e) for child in migration.run_before: child = self.check_key(child, key[0]) if child is not None: try: self.graph.add_dependency(migration, child, key) except NodeNotFoundError as e: # Since we added "key" to the nodes before this implies # "child" is not in there. _reraise_missing_dependency(migration, child, e)
# -*- coding: utf-8 -*- from __future__ import unicode_literals from django.db import models, migrations, DEFAULT_DB_ALIAS, connections from django.db.migrations.recorder import MigrationRecorder import django.db.models.deletion from django.conf import settings connection = connections[DEFAULT_DB_ALIAS] recorder = MigrationRecorder(connection) linaro_django_xmlrpc_applied = False lava_scheduler_app_applied = False for app, name in recorder.applied_migrations(): if app == 'linaro_django_xmlrpc' and name == '0001_initial': linaro_django_xmlrpc_applied = True if app == 'lava_scheduler_app' and name == '0001_initial': lava_scheduler_app_applied = True if not linaro_django_xmlrpc_applied and lava_scheduler_app_applied: recorder.record_applied('linaro_django_xmlrpc', '0001_initial') class Migration(migrations.Migration): dependencies = [ ('auth', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('linaro_django_xmlrpc', '__first__'), ('dashboard_app', '__first__'), ]
def test_loading_squashed_erroneous(self): "Tests loading a complex but erroneous set of squashed migrations" loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) def num_nodes(): plan = set(loader.graph.forwards_plan(('migrations', '7_auto'))) return len(plan - loader.applied_migrations) # Empty database: use squashed migration loader.build_graph() self.assertEqual(num_nodes(), 5) # Starting at 1 or 2 should use the squashed migration too recorder.record_applied("migrations", "1_auto") loader.build_graph() self.assertEqual(num_nodes(), 4) recorder.record_applied("migrations", "2_auto") loader.build_graph() self.assertEqual(num_nodes(), 3) # However, starting at 3 or 4, nonexistent migrations would be needed. msg = ("Migration migrations.6_auto depends on nonexistent node ('migrations', '5_auto'). " "Django tried to replace migration migrations.5_auto with any of " "[migrations.3_squashed_5] but wasn't able to because some of the replaced " "migrations are already applied.") recorder.record_applied("migrations", "3_auto") with self.assertRaisesMessage(NodeNotFoundError, msg): loader.build_graph() recorder.record_applied("migrations", "4_auto") with self.assertRaisesMessage(NodeNotFoundError, msg): loader.build_graph() # Starting at 5 to 7 we are passed the squashed migrations recorder.record_applied("migrations", "5_auto") loader.build_graph() self.assertEqual(num_nodes(), 2) recorder.record_applied("migrations", "6_auto") loader.build_graph() self.assertEqual(num_nodes(), 1) recorder.record_applied("migrations", "7_auto") loader.build_graph() self.assertEqual(num_nodes(), 0)
def test_loading_squashed_complex(self): "Tests loading a complex set of squashed migrations" loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) def num_nodes(): plan = set(loader.graph.forwards_plan(('migrations', '7_auto'))) return len(plan - loader.applied_migrations) # Empty database: use squashed migration loader.build_graph() self.assertEqual(num_nodes(), 5) # Starting at 1 or 2 should use the squashed migration too recorder.record_applied("migrations", "1_auto") loader.build_graph() self.assertEqual(num_nodes(), 4) recorder.record_applied("migrations", "2_auto") loader.build_graph() self.assertEqual(num_nodes(), 3) # However, starting at 3 to 5 cannot use the squashed migration recorder.record_applied("migrations", "3_auto") loader.build_graph() self.assertEqual(num_nodes(), 4) recorder.record_applied("migrations", "4_auto") loader.build_graph() self.assertEqual(num_nodes(), 3) # Starting at 5 to 7 we are passed the squashed migrations recorder.record_applied("migrations", "5_auto") loader.build_graph() self.assertEqual(num_nodes(), 2) recorder.record_applied("migrations", "6_auto") loader.build_graph() self.assertEqual(num_nodes(), 1) recorder.record_applied("migrations", "7_auto") loader.build_graph() self.assertEqual(num_nodes(), 0)