def test_first(self): """ Makes sure the '__first__' migrations build correctly. """ migration_loader = MigrationLoader(connection) self.assertEqual( migration_loader.graph.forwards_plan(("migrations", "second")), [ ("migrations", "thefirst"), ("migrations2", "0001_initial"), ("migrations2", "0002_second"), ("migrations", "second"), ], )
def __init__( self, path=None, ignore_name_contains=None, ignore_name=None, include_apps=None, exclude_apps=None, database=DEFAULT_DB_ALIAS, cache_path=DEFAULT_CACHE_PATH, no_cache=False, only_applied_migrations=False, only_unapplied_migrations=False, exclude_migration_tests=None, quiet=None, warnings_as_errors=False, ): # Store parameters and options self.django_path = path self.ignore_name_contains = ignore_name_contains self.ignore_name = ignore_name or tuple() self.include_apps = include_apps self.exclude_apps = exclude_apps self.exclude_migration_tests = exclude_migration_tests or [] self.database = database or DEFAULT_DB_ALIAS self.cache_path = cache_path or DEFAULT_CACHE_PATH self.no_cache = no_cache self.only_applied_migrations = only_applied_migrations self.only_unapplied_migrations = only_unapplied_migrations self.quiet = quiet or [] self.warnings_as_errors = warnings_as_errors # Initialise counters self.nb_valid = 0 self.nb_ignored = 0 self.nb_warnings = 0 self.nb_erroneous = 0 self.nb_total = 0 # Initialise cache. Read from old, write to new to prune old entries. if self.should_use_cache(): self.old_cache = Cache(self.django_path, self.database, self.cache_path) self.new_cache = Cache(self.django_path, self.database, self.cache_path) self.old_cache.load() # Initialise migrations from django.db.migrations.loader import MigrationLoader self.migration_loader = MigrationLoader( connection=connections[self.database], load=True )
def cli(debug): log_level = logging.INFO if debug: log_level = logging.DEBUG sys.excepthook = lambda t, v, tb: ipdb.post_mortem(tb) coloredlogs.install(level=log_level, fmt=os.getenv("COLOREDLOGS_LOG_FORMAT", DEFAULT_LOG_FORMAT)) connection = connections['default'] loader = MigrationLoader(connection, ignore_no_migrations=True) all_migrations = set(loader.disk_migrations.keys()) if all_migrations != loader.applied_migrations: management.call_command('migrate', noinput=True)
def show_list(self, connection, app_names=None): """ Show a list of all migrations on the system, or only those of some named apps. """ # Load migrations from disk/DB loader = MigrationLoader(connection, ignore_no_migrations=True) recorder = MigrationRecorder(connection) recorded_migrations = recorder.applied_migrations() graph = loader.graph # If we were passed a list of apps, validate it if app_names: self._validate_app_names(loader, app_names) # Otherwise, show all apps in alphabetic order else: app_names = sorted(loader.migrated_apps) # For each app, print its migrations in order from oldest (roots) to # newest (leaves). for app_name in app_names: self.stdout.write(app_name, self.style.MIGRATE_LABEL) shown = set() for node in graph.leaf_nodes(app_name): for plan_node in graph.forwards_plan(node): if plan_node not in shown and plan_node[0] == app_name: # Give it a nice title if it's a squashed one title = plan_node[1] if graph.nodes[plan_node].replaces: title += " (%s squashed migrations)" % len( graph.nodes[plan_node].replaces) applied_migration = loader.applied_migrations.get( plan_node) # Mark it as applied/unapplied if applied_migration: if plan_node in recorded_migrations: output = " [X] %s" % title else: title += " Run 'manage.py migrate' to finish recording." output = " [-] %s" % title if self.verbosity >= 2 and hasattr( applied_migration, "applied"): output += (" (applied at %s)" % applied_migration.applied.strftime( "%Y-%m-%d %H:%M:%S")) self.stdout.write(output) else: self.stdout.write(" [ ] %s" % title) shown.add(plan_node) # If we didn't print anything, then a small message if not shown: self.stdout.write(" (no migrations)", self.style.ERROR)
def test_loading_squashed_erroneous(self): "Tests loading a complex but erroneous set of squashed migrations" loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) def num_nodes(): plan = set(loader.graph.forwards_plan(('migrations', '7_auto'))) return len(plan - loader.applied_migrations) # Empty database: use squashed migration loader.build_graph() self.assertEqual(num_nodes(), 5) # Starting at 1 or 2 should use the squashed migration too recorder.record_applied("migrations", "1_auto") loader.build_graph() self.assertEqual(num_nodes(), 4) recorder.record_applied("migrations", "2_auto") loader.build_graph() self.assertEqual(num_nodes(), 3) # However, starting at 3 or 4, nonexistent migrations would be needed. msg = ("Migration migrations.6_auto depends on nonexistent node ('migrations', '5_auto'). " "Django tried to replace migration migrations.5_auto with any of " "[migrations.3_squashed_5] but wasn't able to because some of the replaced " "migrations are already applied.") recorder.record_applied("migrations", "3_auto") with self.assertRaisesMessage(NodeNotFoundError, msg): loader.build_graph() recorder.record_applied("migrations", "4_auto") with self.assertRaisesMessage(NodeNotFoundError, msg): loader.build_graph() # Starting at 5 to 7 we are passed the squashed migrations recorder.record_applied("migrations", "5_auto") loader.build_graph() self.assertEqual(num_nodes(), 2) recorder.record_applied("migrations", "6_auto") loader.build_graph() self.assertEqual(num_nodes(), 1) recorder.record_applied("migrations", "7_auto") loader.build_graph() self.assertEqual(num_nodes(), 0)
def test_run_before(self): """ Makes sure the loader uses Migration.run_before. """ # Load and test the plan migration_loader = MigrationLoader(connection) self.assertEqual( migration_loader.graph.forwards_plan(("migrations", "0002_second")), [ ("migrations", "0001_initial"), ("migrations", "0003_third"), ("migrations", "0002_second"), ], )
def handle(self, *app_labels, **options): # get supported options. self.verbosity = options['verbosity'] self.dry_run = options['dry_run'] self.merge = options['merge'] self.migration_name = options['name'] self.include_header = options['include_header'] check_changes = options['check_changes'] # validation application labels self.validate_applications(app_labels) # we don't check conflicts as regular makemigrations command. # we don't check if any migrations are applied before their dependencies as regular makemigrations command. # load migrations using same loader as in regular command loader = MigrationLoader(None, ignore_no_migrations=True) from_state = loader.project_state() to_state = ProjectState.from_apps(apps) # overwritten autodetector. They detect only view changes. autodetector = ViewMigrationAutoDetector( from_state, to_state, ) changes = autodetector.changes( graph=loader.graph, trim_to_apps=app_labels or None, convert_apps=app_labels or None, migration_name=self.migration_name, ) # it's copy paste from make migration command if not changes: # No changes? Tell them. if self.verbosity >= 1: if len(app_labels) == 1: self.stdout.write("No changes detected in app '%s'" % app_labels.pop()) elif len(app_labels) > 1: self.stdout.write("No changes detected in apps '%s'" % ("', '".join(app_labels))) else: self.stdout.write("No changes detected") else: self.write_migration_files(changes)
def write_migration(self, migration): loader = MigrationLoader(None, ignore_no_migrations=True) autodetector = MigrationAutodetector( loader.project_state(), ProjectState.from_apps(apps), ) changes = autodetector.arrange_for_graph( changes={'share': [migration]}, graph=loader.graph, ) for m in changes['share']: writer = MigrationWriter(m) with open(writer.path, 'wb') as fp: fp.write(writer.as_string())
def list_migrations(self): """ Returns a tuple of unapplied, current "Unapplied" is a list of unapplied migrations. "Current" is a list of the current migration states for apps with unapplied migrations. Both are tuples of the form (app, migration_name). """ connection = connections[self._database_name] loader = MigrationLoader(connection, ignore_no_migrations=True) unapplied = self._get_unapplied_migrations(loader) currents = self._get_current_migration_state(loader, [u[0] for u in unapplied]) return unapplied, currents
def test_plan_handles_repeated_migrations(self): """ _generate_plan() doesn't readd migrations already in the plan (#29180). """ migration_loader = MigrationLoader(connection) nodes = [("migrations", "0002_second"), ("migrations2", "0001_initial")] self.assertEqual( migration_loader.graph._generate_plan(nodes, at_end=True), [ ("migrations", "0001_initial"), ("migrations", "0002_second"), ("migrations2", "0001_initial"), ], )
def run_required_data_migrations(self): migration_methods = ( ('wagtail.wagtailcore.migrations.0002_initial_data', 'initial_data'), ('wagtail.wagtailcore.migrations.0025_collection_initial_data', 'initial_data'), ) loader = MigrationLoader(connection) for migration, method in migration_methods: if not self.is_migration_applied(loader, migration): print('applying migration {}'.format(migration)) module = importlib.import_module(migration) getattr(module, method)(apps, None)
def test_latest(self): """ Makes sure that __latest__ works correctly. """ # Load and test the plan migration_loader = MigrationLoader(connection) self.assertEqual( migration_loader.graph.forwards_plan( ("migrations", "0001_initial")), [ ("basic", "0001_initial"), ("basic", "0002_second"), ("migrations", "0001_initial"), ], )
def show_list(self, connection): loader = MigrationLoader(connection, ignore_no_migrations=True) app_names = set(settings.PROJECT_APPS) applied_migrations = {(a, m) for a, m in loader.applied_migrations if a in app_names} latest_migrations = defaultdict(lambda: None) for app, migration in applied_migrations: latest_migrations[app] = max(latest_migrations[app], migration) for app, migration in sorted(latest_migrations.iteritems()): print './manage.py migrate {app} {migration}'.format( app=app, migration=migration)
def get_leaf_node_migrations_for_app(app: Any) -> Dict[str, Any]: migration_graph = MigrationLoader(None).graph leaf_migrations = { migration_name: import_module(f"{app.name}.migrations.{migration_name}") for app_name, migration_name in migration_graph.leaf_nodes() if app_name == app.label } if not leaf_migrations: raise Exception(f"Found no migrations for app {app.name}.") count = len(leaf_migrations) if count > 2: raise Exception(f"Too many leaf nodes found for app {app.name}! Found {count}.") return leaf_migrations
def get_objects(): from django.db.migrations.loader import MigrationLoader loader = MigrationLoader(self.connection) for app_config in apps.get_app_configs(): if (app_config.models_module is not None and app_config.label in loader.migrated_apps and app_config.name not in settings.TEST_NON_SERIALIZED_APPS): for model in app_config.get_models(): if (model._meta.can_migrate(self.connection) and router.allow_migrate_model( self.connection.alias, model)): queryset = model._default_manager.using( self.connection.alias, ).order_by( model._meta.pk.name) yield from queryset.iterator()
def test_loading_squashed_complex_multi_apps_partially_applied(self): loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) recorder.record_applied('app1', '1_auto') recorder.record_applied('app1', '2_auto') loader.build_graph() plan = set(loader.graph.forwards_plan(('app1', '4_auto'))) plan = plan - loader.applied_migrations expected_plan = { ('app2', '1_squashed_2'), ('app1', '3_auto'), ('app1', '4_auto'), } self.assertEqual(plan, expected_plan)
def test_loading_squashed_complex_multi_apps_partially_applied(self): loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.record_applied(recorder, "app1", "1_auto") self.record_applied(recorder, "app1", "2_auto") loader.build_graph() plan = set(loader.graph.forwards_plan(("app1", "4_auto"))) plan = plan - loader.applied_migrations.keys() expected_plan = { ("app2", "1_squashed_2"), ("app1", "3_auto"), ("app1", "4_auto"), } self.assertEqual(plan, expected_plan)
def show_plan(self, connection, app_names=None): """ Show all known migrations (or only those of the specified app_names) in the order they will be applied. """ # Load migrations from disk/DB loader = MigrationLoader(connection) graph = loader.graph if app_names: self._validate_app_names(loader, app_names) targets = [ key for key in graph.leaf_nodes() if key[0] in app_names ] else: targets = graph.leaf_nodes() plan = [] seen = set() # Generate the plan for target in targets: for migration in graph.forwards_plan(target): if migration not in seen: node = graph.node_map[migration] plan.append(node) seen.add(migration) # Output def print_deps(node): out = [] for parent in sorted(node.parents): out.append("%s.%s" % parent.key) if out: return " ... (%s)" % ", ".join(out) return "" for node in plan: deps = "" if self.verbosity >= 2: deps = print_deps(node) if node.key in loader.applied_migrations: self.stdout.write("[X] %s.%s%s" % (node.key[0], node.key[1], deps)) else: self.stdout.write("[ ] %s.%s%s" % (node.key[0], node.key[1], deps)) if not plan: self.stdout.write('(no migrations)', self.style.ERROR)
def test_django_17_migrations(self): from django.apps import apps from django.db.migrations.loader import MigrationLoader from django.db.migrations.autodetector import MigrationAutodetector from django.db.migrations.state import ProjectState from django.db.migrations.questioner import MigrationQuestioner app_labels = set(app.label for app in apps.get_app_configs() if app.name.startswith('wagtail.')) for app_label in app_labels: apps.get_app_config(app_label.split('.')[-1]) loader = MigrationLoader(None, ignore_no_migrations=True) conflicts = dict( (app_label, conflict) for app_label, conflict in iteritems(loader.detect_conflicts()) if app_label in app_labels) if conflicts: name_str = "; ".join("%s in %s" % (", ".join(names), app) for app, names in conflicts.items()) self.fail("Conflicting migrations detected (%s)." % name_str) autodetector = MigrationAutodetector( loader.project_state(), ProjectState.from_apps(apps), MigrationQuestioner(specified_apps=app_labels, dry_run=True), ) changes = autodetector.changes( graph=loader.graph, trim_to_apps=app_labels or None, convert_apps=app_labels or None, ) if changes: apps = ', '.join( apps.get_app_config(label).name for label in changes.keys()) migrations = '\n'.join((' {migration}\n{changes}'.format( migration=migration, changes='\n'.join(' {0}'.format(operation.describe()) for operation in migration.operations)) for (_, migrations) in changes.items() for migration in migrations)) self.fail('Model changes with no migrations detected:\n%s' % migrations)
def migrations_have_applied(): """ Check if there are any migrations that haven't been applied yet """ connection = connections[DEFAULT_DB_ALIAS] loader = MigrationLoader(connection) graph = loader.graph # Count unapplied migrations num_unapplied_migrations = 0 for app_name in loader.migrated_apps: for node in graph.leaf_nodes(app_name): for plan_node in graph.forwards_plan(node): if plan_node not in loader.applied_migrations: num_unapplied_migrations += 1 return num_unapplied_migrations == 0
def test_loading_squashed(self): "Tests loading a squashed migration" migration_loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) # Loading with nothing applied should just give us the one node self.assertEqual( len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]), 1, ) # However, fake-apply one migration and it should now use the old two recorder.record_applied("migrations", "0001_initial") migration_loader.build_graph() self.assertEqual( len([x for x in migration_loader.graph.nodes if x[0] == "migrations"]), 2, )
def test_valid(self): """ To support frozen environments, MigrationLoader loads .pyc migrations. """ with self.temporary_migration_module( module='migrations.test_migrations') as migration_dir: # Compile .py files to .pyc files and delete .py files. compileall.compile_dir(migration_dir, force=True, quiet=1, legacy=True) for name in os.listdir(migration_dir): if name.endswith('.py'): os.remove(os.path.join(migration_dir, name)) loader = MigrationLoader(connection) self.assertIn(('migrations', '0001_initial'), loader.disk_migrations)
def setUp(self): from django.db.migrations.executor import MigrationExecutor from django.db.migrations.loader import MigrationLoader from django.db import connection self.migration_executor = MigrationExecutor(connection) self.migration_loader = MigrationLoader(connection) self.state_0001 = self.migration_loader.project_state( ('altersortedmanytomanyfield_tests', '0001_initial')) self.state_0002 = self.migration_loader.project_state( ('altersortedmanytomanyfield_tests', '0002_alter_m2m_fields')) self.state_0001_apps = get_apps_from_state(self.state_0001) self.state_0002_apps = get_apps_from_state(self.state_0002) # Make sure we are at the latest migration when starting the test. with capture_stdout(): call_command('migrate', 'altersortedmanytomanyfield_tests')
def test_loading_squashed_complex(self): "Tests loading a complex set of squashed migrations" loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) def num_nodes(): plan = set(loader.graph.forwards_plan(('migrations', '7_auto'))) return len(plan - loader.applied_migrations) # Empty database: use squashed migration loader.build_graph() self.assertEqual(num_nodes(), 5) # Starting at 1 or 2 should use the squashed migration too recorder.record_applied("migrations", "1_auto") loader.build_graph() self.assertEqual(num_nodes(), 4) recorder.record_applied("migrations", "2_auto") loader.build_graph() self.assertEqual(num_nodes(), 3) # However, starting at 3 to 5 cannot use the squashed migration recorder.record_applied("migrations", "3_auto") loader.build_graph() self.assertEqual(num_nodes(), 4) recorder.record_applied("migrations", "4_auto") loader.build_graph() self.assertEqual(num_nodes(), 3) # Starting at 5 to 7 we are passed the squashed migrations recorder.record_applied("migrations", "5_auto") loader.build_graph() self.assertEqual(num_nodes(), 2) recorder.record_applied("migrations", "6_auto") loader.build_graph() self.assertEqual(num_nodes(), 1) recorder.record_applied("migrations", "7_auto") loader.build_graph() self.assertEqual(num_nodes(), 0) recorder.flush()
def test_invalid(self): """ MigrationLoader reraises ImportErrors caused by "bad magic number" pyc files with a more helpful message. """ with self.temporary_migration_module( module="migrations.test_migrations_bad_pyc") as migration_dir: # The -tpl suffix is to avoid the pyc exclusion in MANIFEST.in. os.rename( os.path.join(migration_dir, "0001_initial.pyc-tpl"), os.path.join(migration_dir, "0001_initial.pyc"), ) msg = ( r"Couldn't import '\w+.migrations.0001_initial' as it appears " "to be a stale .pyc file.") with self.assertRaisesRegex(ImportError, msg): MigrationLoader(connection)
def handle(self, out_folder, **options): loader = MigrationLoader(connection=connections[DEFAULT_DB_ALIAS]) # Prevent override apps folders if out_folder == '.': out_folder = self.out_folder_default # Call sqlmigrate for all migrations of each app bar = progressbar.ProgressBar() migrations = self.get_all_migrations(loader) for i in bar(range(len(migrations))): migration = migrations[i] self.write_sqlmigrate(out_folder, migration, options['no_folder_per_app']) self.stdout.write( self.style.SUCCESS(self.command_sucessfull_finish % (os.path.abspath(out_folder))))
def forward_func(apps, schema_editor): reader = ResourceSnapshotReader(Migration.snapshot_json) snapshot = reader.read() last_snapshot = None if Migration.dependencies: loader = MigrationLoader(None, ignore_no_migrations=True) last_migration = loader.disk_migrations[Migration.dependencies[0]] last_reader = ResourceSnapshotReader(last_migration.snapshot_json) last_snapshot = last_reader.read() differ = SnapshotDiffer(last_snapshot, snapshot) operations = differ.diff_operations() migration = ResourceMigration.get_migration(operations) migration.apply()
def exportOneExperiment(cls, experiment): # Read data from database all = HIT.objects.count() submitted = HIT.objects.filter(data__status='Submitted').count() accepted = HIT.objects.filter(data__status='Accepted').count() rejected = HIT.objects.filter(data__status='Rejected').count() status = ';'.join([i for _, i in MigrationLoader(connection=connection, ignore_no_migrations=True).graph.leaf_nodes('users')]) # Put data on first sheet book = Workbook() #write_only=True metasheet = book.active #book.create_sheet("Experiment Information") metasheet.title = "Experiment Information" metasheet.append(("Experiment ID", -1)) #experiment.id metasheet.append(("Start Date", -2)) #experiment.start metasheet.append(("End Date", -3 or "Ongoing")) #experiment.end metasheet.append(("Export Date", datetime.now())) metasheet.append((None, None)) metasheet.append(("HITs Created", -4)) #experiment.hits metasheet.append(("HITs Attempted", all)) metasheet.append(("HITs Completed", submitted + accepted + rejected)) metasheet.append(("HITs Accepted", accepted)) metasheet.append(("HITs Rejected", rejected)) metasheet.append((None, None)) metasheet.append(("Migration Status", status)) metasheet.sheet_state = 'hidden' # Create the other sheets questions = cls.exportSheet(book, Question.objects.all().order_by('-isFinal', 'id'), 'id','text','isFinal','hit','mergeParent', mergeParent=None ) answers = cls.exportSheet(book, Answer.objects.all().order_by('-isFinal', 'id'), 'id','text','isFinal','question','hit', question=questions, ) attributes = cls.exportSheet(book, Attribute.objects.all().order_by('count'), 'word','count','weight','question:answer.question','answer', question=questions, answer=answers, ) return save_virtual_workbook(book)
def handle(self, *args, **options): json_file = options["migration_json"] if not json_file: sys.stderr.write("please provide a migration json file name\n") exit(1) json_path = getattr(settings, "BK_IAM_MIGRATION_JSON_PATH", "support-files/iam/") file_path = os.path.join(settings.BASE_DIR, json_path, json_file) if not os.path.exists(file_path): sys.stderr.write("{} is not exist.\n".format(file_path)) exit(1) loader = MigrationLoader(None, ignore_no_migrations=False) migration_leaf = loader.graph.leaf_nodes(conf.MIGRATION_APP_NAME) is_initial = True last_migration_name = None if migration_leaf: is_initial = False last_migration_name = migration_leaf[0][1] sys.stdout.write("is initial migration: {}\n".format(is_initial)) sys.stdout.write("last migration: {}\n".format(last_migration_name)) migration_name = self.migration_name(last_migration_name) migration_file = "{}.py".format( os.path.join(apps.get_app_config(conf.MIGRATION_APP_NAME).path, "migrations", migration_name) ) with codecs.open(migration_file, mode="w", encoding="utf-8") as fp: template = Engine.get_default().from_string(migration_template) fp.write( template.render( Context( { "migration_json": json_file, "app_label": IAMMigrationConfig.name, "initial": is_initial, "last_migration_name": last_migration_name, } ) ) )
def test_loading_squashed_complex(self): "Tests loading a complex set of squashed migrations" loader = MigrationLoader(connection) recorder = MigrationRecorder(connection) self.addCleanup(recorder.flush) def num_nodes(): plan = set(loader.graph.forwards_plan(('migrations', '7_auto'))) return len(plan - loader.applied_migrations.keys()) # Empty database: use squashed migration loader.build_graph() self.assertEqual(num_nodes(), 5) # Starting at 1 or 2 should use the squashed migration too self.record_applied(recorder, 'migrations', '1_auto') loader.build_graph() self.assertEqual(num_nodes(), 4) self.record_applied(recorder, 'migrations', '2_auto') loader.build_graph() self.assertEqual(num_nodes(), 3) # However, starting at 3 to 5 cannot use the squashed migration self.record_applied(recorder, 'migrations', '3_auto') loader.build_graph() self.assertEqual(num_nodes(), 4) self.record_applied(recorder, 'migrations', '4_auto') loader.build_graph() self.assertEqual(num_nodes(), 3) # Starting at 5 to 7 we are past the squashed migrations. self.record_applied(recorder, 'migrations', '5_auto') loader.build_graph() self.assertEqual(num_nodes(), 2) self.record_applied(recorder, 'migrations', '6_auto') loader.build_graph() self.assertEqual(num_nodes(), 1) self.record_applied(recorder, 'migrations', '7_auto') loader.build_graph() self.assertEqual(num_nodes(), 0)