def test_name_match(self):
"Tests prefix name matching"
migration_loader = MigrationLoader(connection)
self.assertEqual(migration_loader.get_migration_by_prefix("migrations", "0001").name, "0001_initial")
with self.assertRaises(AmbiguityError):
migration_loader.get_migration_by_prefix("migrations", "0")
with self.assertRaises(KeyError):
migration_loader.get_migration_by_prefix("migrations", "blarg")
def test_name_match(self):
"Tests prefix name matching"
migration_loader = MigrationLoader(connection)
self.assertEqual(
migration_loader.get_migration_by_prefix("migrations", "0001").name,
"0001_initial",
)
with self.assertRaises(AmbiguityError):
migration_loader.get_migration_by_prefix("migrations", "0")
with self.assertRaises(KeyError):
migration_loader.get_migration_by_prefix("migrations", "blarg")
def test_name_match(self):
"Tests prefix name matching"
migration_loader = MigrationLoader(connection)
self.assertEqual(
migration_loader.get_migration_by_prefix("migrations",
"0001").name,
"0001_initial",
)
msg = "There is more than one migration for 'migrations' with the prefix '0'"
with self.assertRaisesMessage(AmbiguityError, msg):
migration_loader.get_migration_by_prefix("migrations", "0")
msg = "There is no migration for 'migrations' with the prefix 'blarg'"
with self.assertRaisesMessage(KeyError, msg):
migration_loader.get_migration_by_prefix("migrations", "blarg")
def _get_apps_for_migration(self, migration_states):
loader = MigrationLoader(connection)
full_names = []
for app_name, migration_name in migration_states:
if migration_name != 'zero':
migration_name = loader.get_migration_by_prefix(
app_name, migration_name).name
full_names.append((app_name, migration_name))
state = loader.project_state(full_names)
return state.apps
def _get_apps_for_migration(self, migration_states):
loader = MigrationLoader(connection)
full_names = []
for app_name, migration_name in migration_states:
if migration_name != 'zero':
migration_name = loader.get_migration_by_prefix(
app_name, migration_name).name
full_names.append((app_name, migration_name))
state = loader.project_state(full_names)
return state.apps
def _get_apps_for_migration(self, migration_states):
loader = MigrationLoader(connection)
full_names = []
for app_name, migration_name in migration_states:
migration_name = loader.get_migration_by_prefix(app_name, migration_name).name
full_names.append((app_name, migration_name))
state = loader.project_state(full_names)
if django.VERSION < (1, 8):
state.render()
return state.apps
def _get_apps_for_migration(self, migration_states):
loader = MigrationLoader(connection)
full_names = []
for app_name, migration_name in migration_states:
migration_name = loader.get_migration_by_prefix(
app_name, migration_name).name
full_names.append((app_name, migration_name))
state = loader.project_state(full_names)
if django.VERSION < (1, 8):
state.render()
return state.apps
def handle(self, *args, **options):
# Get the database we're operating from
connection = connections[options["database"]]
# Load up an loader to get all the migration data, but don't replace
# migrations.
loader = MigrationLoader(connection, replace_migrations=False)
# Resolve command-line arguments into a migration
app_label, migration_name = options["app_label"], options[
"migration_name"]
# Validate app_label
try:
apps.get_app_config(app_label)
except LookupError as err:
raise CommandError(str(err))
if app_label not in loader.migrated_apps:
raise CommandError("App '%s' does not have migrations" % app_label)
try:
migration = loader.get_migration_by_prefix(app_label,
migration_name)
except AmbiguityError:
raise CommandError(
"More than one migration matches '%s' in app '%s'. Please be more specific."
% (migration_name, app_label))
except KeyError:
raise CommandError(
"Cannot find a migration matching '%s' from app '%s'. Is it in INSTALLED_APPS?"
% (migration_name, app_label))
target = (app_label, migration.name)
# Show begin/end around output for atomic migrations, if the database
# supports transactional DDL.
self.output_transaction = (migration.atomic
and connection.features.can_rollback_ddl)
# Make a plan that represents just the requested migrations and show SQL
# for it
plan = [(loader.graph.nodes[target], options["backwards"])]
sql_statements = loader.collect_sql(plan)
if not sql_statements and options["verbosity"] >= 1:
self.stderr.write("No operations found.")
return "\n".join(sql_statements)
def test_sqlmigrate(self):
from django.db import connection
if django.VERSION < (3, 0): # for dj22
from django.db.migrations.executor import MigrationExecutor
executor = MigrationExecutor(connection)
loader = executor.loader
collect_sql = executor.collect_sql
else:
from django.db.migrations.loader import MigrationLoader
loader = MigrationLoader(connection)
collect_sql = loader.collect_sql
app_label, migration_name = 'testapp', '0001'
migration = loader.get_migration_by_prefix(app_label, migration_name)
target = (app_label, migration.name)
plan = [(loader.graph.nodes[target], False)]
sql_statements = collect_sql(plan)
print('\n'.join(sql_statements))
assert sql_statements # It doesn't matter what SQL is generated.
def handle(self, **options):
self.verbosity = options.get('verbosity')
self.interactive = options.get('interactive')
app_label = options['app_label']
migration_name = options['migration_name']
no_optimize = options['no_optimize']
# Load the current graph state, check the app and migration they asked for exists
loader = MigrationLoader(connections[DEFAULT_DB_ALIAS])
if app_label not in loader.migrated_apps:
raise CommandError(
"App '%s' does not have migrations (so squashmigrations on "
"it makes no sense)" % app_label)
try:
migration = loader.get_migration_by_prefix(app_label,
migration_name)
except AmbiguityError:
raise CommandError(
"More than one migration matches '%s' in app '%s'. Please be "
"more specific." % (migration_name, app_label))
except KeyError:
raise CommandError(
"Cannot find a migration matching '%s' from app '%s'." %
(migration_name, app_label))
# Work out the list of predecessor migrations
migrations_to_squash = [
loader.get_migration(al, mn)
for al, mn in loader.graph.forwards_plan((migration.app_label,
migration.name))
if al == migration.app_label
]
# Tell them what we're doing and optionally ask if we should proceed
if self.verbosity > 0 or self.interactive:
self.stdout.write(
self.style.MIGRATE_HEADING(
"Will squash the following migrations:"))
for migration in migrations_to_squash:
self.stdout.write(" - %s" % migration.name)
if self.interactive:
answer = None
while not answer or answer not in "yn":
answer = six.moves.input("Do you wish to proceed? [yN] ")
if not answer:
answer = "n"
break
else:
answer = answer[0].lower()
if answer != "y":
return
# Load the operations from all those migrations and concat together,
# along with collecting external dependencies and detecting
# double-squashing
operations = []
dependencies = set()
for smigration in migrations_to_squash:
if smigration.replaces:
raise CommandError(
"You cannot squash squashed migrations! Please transition "
"it to a normal migration first: "
"https://docs.djangoproject.com/en/%s/topics/migrations/#squashing-migrations"
% get_docs_version())
operations.extend(smigration.operations)
for dependency in smigration.dependencies:
if isinstance(dependency, SwappableTuple):
if settings.AUTH_USER_MODEL == dependency.setting:
dependencies.add(("__setting__", "AUTH_USER_MODEL"))
else:
dependencies.add(dependency)
elif dependency[0] != smigration.app_label:
dependencies.add(dependency)
if no_optimize:
if self.verbosity > 0:
self.stdout.write(
self.style.MIGRATE_HEADING("(Skipping optimization.)"))
new_operations = operations
else:
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("Optimizing..."))
optimizer = MigrationOptimizer()
new_operations = optimizer.optimize(operations,
migration.app_label)
if self.verbosity > 0:
if len(new_operations) == len(operations):
self.stdout.write(" No optimizations possible.")
else:
self.stdout.write(
" Optimized from %s operations to %s operations." %
(len(operations), len(new_operations)))
# Work out the value of replaces (any squashed ones we're re-squashing)
# need to feed their replaces into ours
replaces = []
for migration in migrations_to_squash:
if migration.replaces:
replaces.extend(migration.replaces)
else:
replaces.append((migration.app_label, migration.name))
# Make a new migration with those operations
subclass = type(
"Migration", (migrations.Migration, ), {
"dependencies": dependencies,
"operations": new_operations,
"replaces": replaces,
"initial": True,
})
new_migration = subclass("0001_squashed_%s" % migration.name,
app_label)
# Write out the new migration file
writer = MigrationWriter(new_migration)
with open(writer.path, "wb") as fh:
fh.write(writer.as_string())
if self.verbosity > 0:
self.stdout.write(
self.style.MIGRATE_HEADING(
"Created new squashed migration %s" % writer.path))
self.stdout.write(
" You should commit this migration but leave the old ones in place;"
)
self.stdout.write(
" the new migration will be used for new installs. Once you are sure"
)
self.stdout.write(
" all instances of the codebase have applied the migrations you squashed,"
)
self.stdout.write(" you can delete them.")
if writer.needs_manual_porting:
self.stdout.write(
self.style.MIGRATE_HEADING("Manual porting required"))
self.stdout.write(
" Your migrations contained functions that must be manually copied over,"
)
self.stdout.write(
" as we could not safely copy their implementation.")
self.stdout.write(
" See the comment at the top of the squashed migration for details."
)
def handle(self, **options):
self.verbosity = options.get('verbosity')
self.interactive = options.get('interactive')
app_label = options['app_label']
migration_name = options['migration_name']
no_optimize = options['no_optimize']
# Load the current graph state, check the app and migration they asked for exists
loader = MigrationLoader(connections[DEFAULT_DB_ALIAS])
if app_label not in loader.migrated_apps:
raise CommandError(
"App '%s' does not have migrations (so squashmigrations on "
"it makes no sense)" % app_label
)
try:
migration = loader.get_migration_by_prefix(app_label, migration_name)
except AmbiguityError:
raise CommandError(
"More than one migration matches '%s' in app '%s'. Please be "
"more specific." % (migration_name, app_label)
)
except KeyError:
raise CommandError(
"Cannot find a migration matching '%s' from app '%s'." %
(migration_name, app_label)
)
# Work out the list of predecessor migrations
migrations_to_squash = [
loader.get_migration(al, mn)
for al, mn in loader.graph.forwards_plan((migration.app_label, migration.name))
if al == migration.app_label
]
# Tell them what we're doing and optionally ask if we should proceed
if self.verbosity > 0 or self.interactive:
self.stdout.write(self.style.MIGRATE_HEADING("Will squash the following migrations:"))
for migration in migrations_to_squash:
self.stdout.write(" - %s" % migration.name)
if self.interactive:
answer = None
while not answer or answer not in "yn":
answer = six.moves.input("Do you wish to proceed? [yN] ")
if not answer:
answer = "n"
break
else:
answer = answer[0].lower()
if answer != "y":
return
# Load the operations from all those migrations and concat together,
# along with collecting external dependencies and detecting
# double-squashing
operations = []
dependencies = set()
for smigration in migrations_to_squash:
if smigration.replaces:
raise CommandError(
"You cannot squash squashed migrations! Please transition "
"it to a normal migration first: "
"https://docs.djangoproject.com/en/%s/topics/migrations/#squashing-migrations" % get_docs_version()
)
operations.extend(smigration.operations)
for dependency in smigration.dependencies:
if isinstance(dependency, SwappableTuple):
if settings.AUTH_USER_MODEL == dependency.setting:
dependencies.add(("__setting__", "AUTH_USER_MODEL"))
else:
dependencies.add(dependency)
elif dependency[0] != smigration.app_label:
dependencies.add(dependency)
if no_optimize:
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("(Skipping optimization.)"))
new_operations = operations
else:
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("Optimizing..."))
optimizer = MigrationOptimizer()
new_operations = optimizer.optimize(operations, migration.app_label)
if self.verbosity > 0:
if len(new_operations) == len(operations):
self.stdout.write(" No optimizations possible.")
else:
self.stdout.write(
" Optimized from %s operations to %s operations." %
(len(operations), len(new_operations))
)
# Work out the value of replaces (any squashed ones we're re-squashing)
# need to feed their replaces into ours
replaces = []
for migration in migrations_to_squash:
if migration.replaces:
replaces.extend(migration.replaces)
else:
replaces.append((migration.app_label, migration.name))
# Make a new migration with those operations
subclass = type("Migration", (migrations.Migration, ), {
"dependencies": dependencies,
"operations": new_operations,
"replaces": replaces,
"initial": True,
})
new_migration = subclass("0001_squashed_%s" % migration.name, app_label)
# Write out the new migration file
writer = MigrationWriter(new_migration)
with open(writer.path, "wb") as fh:
fh.write(writer.as_string())
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("Created new squashed migration %s" % writer.path))
self.stdout.write(" You should commit this migration but leave the old ones in place;")
self.stdout.write(" the new migration will be used for new installs. Once you are sure")
self.stdout.write(" all instances of the codebase have applied the migrations you squashed,")
self.stdout.write(" you can delete them.")
if writer.needs_manual_porting:
self.stdout.write(self.style.MIGRATE_HEADING("Manual porting required"))
self.stdout.write(" Your migrations contained functions that must be manually copied over,")
self.stdout.write(" as we could not safely copy their implementation.")
self.stdout.write(" See the comment at the top of the squashed migration for details.")
def handle(self, *args, **options):
verbosity = options["verbosity"]
app_label = options["app_label"]
migration_name = options["migration_name"]
check = options["check"]
# Validate app_label.
try:
apps.get_app_config(app_label)
except LookupError as err:
raise CommandError(str(err))
# Load the current graph state.
loader = MigrationLoader(None)
if app_label not in loader.migrated_apps:
raise CommandError(f"App '{app_label}' does not have migrations.")
# Find a migration.
try:
migration = loader.get_migration_by_prefix(app_label, migration_name)
except AmbiguityError:
raise CommandError(
f"More than one migration matches '{migration_name}' in app "
f"'{app_label}'. Please be more specific."
)
except KeyError:
raise CommandError(
f"Cannot find a migration matching '{migration_name}' from app "
f"'{app_label}'."
)
# Optimize the migration.
optimizer = MigrationOptimizer()
new_operations = optimizer.optimize(migration.operations, migration.app_label)
if len(migration.operations) == len(new_operations):
if verbosity > 0:
self.stdout.write("No optimizations possible.")
return
else:
if verbosity > 0:
self.stdout.write(
"Optimizing from %d operations to %d operations."
% (len(migration.operations), len(new_operations))
)
if check:
sys.exit(1)
# Set the new migration optimizations.
migration.operations = new_operations
# Write out the optimized migration file.
writer = MigrationWriter(migration)
migration_file_string = writer.as_string()
if writer.needs_manual_porting:
if migration.replaces:
raise CommandError(
"Migration will require manual porting but is already a squashed "
"migration.\nTransition to a normal migration first: "
"https://docs.djangoproject.com/en/%s/topics/migrations/"
"#squashing-migrations" % get_docs_version()
)
# Make a new migration with those operations.
subclass = type(
"Migration",
(migrations.Migration,),
{
"dependencies": migration.dependencies,
"operations": new_operations,
"replaces": [(migration.app_label, migration.name)],
},
)
optimized_migration_name = "%s_optimized" % migration.name
optimized_migration = subclass(optimized_migration_name, app_label)
writer = MigrationWriter(optimized_migration)
migration_file_string = writer.as_string()
if verbosity > 0:
self.stdout.write(
self.style.MIGRATE_HEADING("Manual porting required") + "\n"
" Your migrations contained functions that must be manually "
"copied over,\n"
" as we could not safely copy their implementation.\n"
" See the comment at the top of the optimized migration for "
"details."
)
if shutil.which("black"):
self.stdout.write(
self.style.WARNING(
"Optimized migration couldn't be formatted using the "
'"black" command. You can call it manually.'
)
)
with open(writer.path, "w", encoding="utf-8") as fh:
fh.write(migration_file_string)
run_formatters([writer.path])
if verbosity > 0:
self.stdout.write(
self.style.MIGRATE_HEADING(f"Optimized migration {writer.path}")
)
class MigrationLinter(object):
def __init__(
self,
path=None,
ignore_name_contains=None,
ignore_name=None,
include_name_contains=None,
include_name=None,
include_apps=None,
exclude_apps=None,
database=DEFAULT_DB_ALIAS,
cache_path=DEFAULT_CACHE_PATH,
no_cache=False,
only_applied_migrations=False,
only_unapplied_migrations=False,
exclude_migration_tests=None,
quiet=None,
warnings_as_errors=False,
no_output=False,
):
# Store parameters and options
self.django_path = path
self.ignore_name_contains = ignore_name_contains
self.ignore_name = ignore_name or tuple()
self.include_name_contains = include_name_contains
self.include_name = include_name or tuple()
self.include_apps = include_apps
self.exclude_apps = exclude_apps
self.exclude_migration_tests = exclude_migration_tests or []
self.database = database or DEFAULT_DB_ALIAS
self.cache_path = cache_path or DEFAULT_CACHE_PATH
self.no_cache = no_cache
self.only_applied_migrations = only_applied_migrations
self.only_unapplied_migrations = only_unapplied_migrations
self.quiet = quiet or []
self.warnings_as_errors = warnings_as_errors
self.no_output = no_output
# Initialise counters
self.reset_counters()
# Initialise cache. Read from old, write to new to prune old entries.
if self.should_use_cache():
self.old_cache = Cache(self.django_path, self.database, self.cache_path)
self.new_cache = Cache(self.django_path, self.database, self.cache_path)
self.old_cache.load()
# Initialise migrations
from django.db.migrations.loader import MigrationLoader
self.migration_loader = MigrationLoader(
connection=connections[self.database], load=True
)
def reset_counters(self):
self.nb_valid = 0
self.nb_ignored = 0
self.nb_warnings = 0
self.nb_erroneous = 0
self.nb_total = 0
def should_use_cache(self):
return self.django_path and not self.no_cache
def lint_all_migrations(
self,
app_label=None,
migration_name=None,
git_commit_id=None,
migrations_file_path=None,
):
# Collect migrations
migrations_list = self.read_migrations_list(migrations_file_path)
if git_commit_id:
migrations = self._gather_migrations_git(git_commit_id, migrations_list)
else:
migrations = self._gather_all_migrations(migrations_list)
# Lint those migrations
sorted_migrations = sorted(
migrations, key=lambda migration: (migration.app_label, migration.name)
)
specific_target_migration = (
self.migration_loader.get_migration_by_prefix(app_label, migration_name)
if app_label and migration_name
else None
)
for m in sorted_migrations:
if app_label and migration_name:
if m == specific_target_migration:
self.lint_migration(m)
elif app_label:
if m.app_label == app_label:
self.lint_migration(m)
else:
self.lint_migration(m)
if self.should_use_cache():
self.new_cache.save()
def lint_migration(self, migration):
app_label = migration.app_label
migration_name = migration.name
operations = migration.operations
self.nb_total += 1
md5hash = self.get_migration_hash(app_label, migration_name)
if self.should_ignore_migration(app_label, migration_name, operations):
self.print_linting_msg(
app_label, migration_name, "IGNORE", MessageType.IGNORE
)
self.nb_ignored += 1
return
if self.should_use_cache() and md5hash in self.old_cache:
self.lint_cached_migration(app_label, migration_name, md5hash)
return
sql_statements = self.get_sql(app_label, migration_name)
errors, ignored, warnings = analyse_sql_statements(
sql_statements,
settings.DATABASES[self.database]["ENGINE"],
self.exclude_migration_tests,
)
err, ignored_data, warnings_data = self.analyse_data_migration(migration)
if err:
errors += err
if ignored_data:
ignored += ignored_data
if warnings_data:
warnings += warnings_data
if self.warnings_as_errors:
errors += warnings
warnings = []
# Fixme: have a more generic approach to handling errors/warnings/ignored/ok?
if errors:
self.print_linting_msg(app_label, migration_name, "ERR", MessageType.ERROR)
self.nb_erroneous += 1
self.print_errors(errors)
if warnings:
self.print_warnings(warnings)
value_to_cache = {"result": "ERR", "errors": errors, "warnings": warnings}
elif warnings:
self.print_linting_msg(
app_label, migration_name, "WARNING", MessageType.WARNING
)
self.nb_warnings += 1
self.print_warnings(warnings)
value_to_cache = {"result": "WARNING", "warnings": warnings}
# Fixme: not displaying ignored errors, when
else:
if ignored:
self.print_linting_msg(
app_label, migration_name, "OK (ignored)", MessageType.IGNORE
)
self.print_errors(ignored)
else:
self.print_linting_msg(app_label, migration_name, "OK", MessageType.OK)
self.nb_valid += 1
value_to_cache = {"result": "OK"}
if self.should_use_cache():
self.new_cache[md5hash] = value_to_cache
@staticmethod
def get_migration_hash(app_label, migration_name):
hash_md5 = hashlib.md5()
with open(get_migration_abspath(app_label, migration_name), "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def lint_cached_migration(self, app_label, migration_name, md5hash):
cached_value = self.old_cache[md5hash]
if cached_value["result"] == "IGNORE":
self.print_linting_msg(
app_label, migration_name, "IGNORE (cached)", MessageType.IGNORE
)
self.nb_ignored += 1
elif cached_value["result"] == "OK":
self.print_linting_msg(
app_label, migration_name, "OK (cached)", MessageType.OK
)
self.nb_valid += 1
elif cached_value["result"] == "WARNING":
self.print_linting_msg(
app_label, migration_name, "WARNING (cached)", MessageType.WARNING
)
self.nb_warnings += 1
self.print_warnings(cached_value["warnings"])
else:
self.print_linting_msg(
app_label, migration_name, "ERR (cached)", MessageType.ERROR
)
self.nb_erroneous += 1
if "errors" in cached_value:
self.print_errors(cached_value["errors"])
if "warnings" in cached_value and cached_value["warnings"]:
self.print_warnings(cached_value["warnings"])
self.new_cache[md5hash] = cached_value
def print_linting_msg(self, app_label, migration_name, msg, lint_result):
if lint_result.value in self.quiet:
return
if not self.no_output:
print("({0}, {1})... {2}".format(app_label, migration_name, msg))
def print_errors(self, errors):
if MessageType.ERROR.value in self.quiet:
return
for err in errors:
error_str = "\t{0}".format(err["msg"])
if err.get("table"):
error_str += " (table: {0}".format(err["table"])
if err.get("column"):
error_str += ", column: {0}".format(err["column"])
error_str += ")"
if not self.no_output:
print(error_str)
def print_warnings(self, warnings):
if MessageType.WARNING.value in self.quiet:
return
for warning_details in warnings:
warn_str = "\t{}".format(warning_details["msg"])
if not self.no_output:
print(warn_str)
def print_summary(self):
if self.no_output:
return
print("*** Summary ***")
print("Valid migrations: {}/{}".format(self.nb_valid, self.nb_total))
print("Erroneous migrations: {}/{}".format(self.nb_erroneous, self.nb_total))
print("Migrations with warnings: {}/{}".format(self.nb_warnings, self.nb_total))
print("Ignored migrations: {}/{}".format(self.nb_ignored, self.nb_total))
@property
def has_errors(self):
return self.nb_erroneous > 0
def get_sql(self, app_label, migration_name):
logger.info(
"Calling sqlmigrate command {} {}".format(app_label, migration_name)
)
dev_null = open(os.devnull, "w")
try:
sql_statement = call_command(
"sqlmigrate",
app_label,
migration_name,
database=self.database,
stdout=dev_null,
)
except (ValueError, ProgrammingError):
logger.warning(
(
"Error while executing sqlmigrate on (%s, %s). "
"Continuing execution with empty SQL."
),
app_label,
migration_name,
)
sql_statement = ""
return sql_statement.splitlines()
@staticmethod
def is_migration_file(filename):
from django.db.migrations.loader import MIGRATIONS_MODULE_NAME
return (
re.search(r"/{0}/.*\.py".format(MIGRATIONS_MODULE_NAME), filename)
and "__init__" not in filename
)
@classmethod
def read_migrations_list(cls, migrations_file_path):
"""
Returning an empty list is different from returning None here.
None: no file was specified and we should consider all migrations
Empty list: no migration found in the file and we should consider no migration
"""
if not migrations_file_path:
return None
migrations = []
try:
with open(migrations_file_path, "r") as file:
for line in file:
if cls.is_migration_file(line):
app_label, name = split_migration_path(line)
migrations.append((app_label, name))
except IOError:
logger.exception("Migrations list path not found %s", migrations_file_path)
raise Exception("Error while reading migrations list file")
if not migrations:
logger.info(
"No valid migration paths found in the migrations file %s",
migrations_file_path,
)
return migrations
def _gather_migrations_git(self, git_commit_id, migrations_list=None):
migrations = []
# Get changes since specified commit
git_diff_command = (
"cd {0} && git diff --relative --name-only --diff-filter=AR {1}"
).format(self.django_path, git_commit_id)
logger.info("Executing {0}".format(git_diff_command))
diff_process = Popen(git_diff_command, shell=True, stdout=PIPE, stderr=PIPE)
for line in map(clean_bytes_to_str, diff_process.stdout.readlines()):
# Only gather lines that include added migrations
if self.is_migration_file(line):
app_label, name = split_migration_path(line)
if migrations_list is None or (app_label, name) in migrations_list:
if (app_label, name) in self.migration_loader.disk_migrations:
migration = self.migration_loader.disk_migrations[
app_label, name
]
migrations.append(migration)
else:
logger.info(
"Found migration file (%s, %s) "
"that is not present in loaded migration.",
app_label,
name,
)
diff_process.wait()
if diff_process.returncode != 0:
output = []
for line in map(clean_bytes_to_str, diff_process.stderr.readlines()):
output.append(line)
logger.error("Error while git diff command:\n{}".format("".join(output)))
raise Exception("Error while executing git diff command")
return migrations
def _gather_all_migrations(self, migrations_list=None):
for (
(app_label, name),
migration,
) in self.migration_loader.disk_migrations.items():
if app_label not in DJANGO_APPS_WITH_MIGRATIONS: # Prune Django apps
if migrations_list is None or (app_label, name) in migrations_list:
yield migration
def should_ignore_migration(self, app_label, migration_name, operations=()):
return (
(self.include_apps and app_label not in self.include_apps)
or (self.exclude_apps and app_label in self.exclude_apps)
or any(isinstance(o, IgnoreMigration) for o in operations)
or (
self.ignore_name_contains
and self.ignore_name_contains in migration_name
)
or (
self.include_name_contains
and self.include_name_contains not in migration_name
)
or (migration_name in self.ignore_name)
or (self.include_name and migration_name not in self.include_name)
or (
self.only_applied_migrations
and (app_label, migration_name)
not in self.migration_loader.applied_migrations
)
or (
self.only_unapplied_migrations
and (app_label, migration_name)
in self.migration_loader.applied_migrations
)
)
def analyse_data_migration(self, migration):
errors = []
ignored = []
warnings = []
for operation in migration.operations:
if isinstance(operation, RunPython):
op_errors, op_ignored, op_warnings = self.lint_runpython(operation)
elif isinstance(operation, RunSQL):
op_errors, op_ignored, op_warnings = self.lint_runsql(operation)
else:
op_errors, op_ignored, op_warnings = [], [], []
if op_errors:
errors += op_errors
if op_ignored:
ignored += op_ignored
if op_warnings:
warnings += op_warnings
return errors, ignored, warnings
def lint_runpython(self, runpython):
function_name = runpython.code.__name__
error = []
ignored = []
warning = []
# Detect warning on missing reverse operation
if not runpython.reversible:
issue = {
"code": "RUNPYTHON_REVERSIBLE",
"msg": "'{}': RunPython data migration is not reversible".format(
function_name
),
}
if issue["code"] in self.exclude_migration_tests:
ignored.append(issue)
else:
warning.append(issue)
# Detect warning for argument naming convention
args_spec = (
inspect.getargspec(runpython.code)
if PY2
else inspect.getfullargspec(runpython.code)
)
if tuple(args_spec.args) != EXPECTED_DATA_MIGRATION_ARGS:
issue = {
"code": "RUNPYTHON_ARGS_NAMING_CONVENTION",
"msg": (
"'{}': By convention, "
"RunPython names the two arguments: apps, schema_editor"
).format(function_name),
}
if issue["code"] in self.exclude_migration_tests:
ignored.append(issue)
else:
warning.append(issue)
# Detect wrong model imports
# Forward
issues = self.get_runpython_model_import_issues(runpython.code)
for issue in issues:
if issue["code"] in self.exclude_migration_tests:
ignored.append(issue)
else:
error.append(issue)
# Backward
if runpython.reversible:
issues = self.get_runpython_model_import_issues(runpython.reverse_code)
for issue in issues:
if issue and issue["code"] in self.exclude_migration_tests:
ignored.append(issue)
else:
error.append(issue)
# Detect warning if model variable name is not the same as model class
issues = self.get_runpython_model_variable_naming_issues(runpython.code)
for issue in issues:
if issue and issue["code"] in self.exclude_migration_tests:
ignored.append(issue)
else:
warning.append(issue)
if runpython.reversible:
issues = self.get_runpython_model_variable_naming_issues(
runpython.reverse_code
)
for issue in issues:
if issue and issue["code"] in self.exclude_migration_tests:
ignored.append(issue)
else:
warning.append(issue)
return error, ignored, warning
@staticmethod
def get_runpython_model_import_issues(code):
model_object_regex = re.compile(r"[^a-zA-Z]?([a-zA-Z0-9]+?)\.objects")
function_name = code.__name__
source_code = inspect.getsource(code)
called_models = model_object_regex.findall(source_code)
issues = []
for model in called_models:
has_get_model_call = (
re.search(
r"{}.*= +\w+\.get_model\(".format(model),
source_code,
)
is not None
)
if not has_get_model_call:
issues.append(
{
"code": "RUNPYTHON_MODEL_IMPORT",
"msg": (
"'{}': Could not find an 'apps.get_model(\"...\", \"{}\")' "
"call. Importing the model directly is incorrect for "
"data migrations."
).format(function_name, model),
}
)
return issues
@staticmethod
def get_runpython_model_variable_naming_issues(code):
model_object_regex = re.compile(r"[^a-zA-Z]?([a-zA-Z0-9]+?)\.objects")
function_name = code.__name__
source_code = inspect.getsource(code)
called_models = model_object_regex.findall(source_code)
issues = []
for model in called_models:
has_same_model_name = (
re.search(
r"{model}.*= +\w+?\.get_model\([^)]+?\.{model}.*?\)".format(
model=model
),
source_code,
re.MULTILINE | re.DOTALL,
)
is not None
or re.search(
r"{model}.*= +\w+?\.get_model\([^)]+?,[^)]*?{model}.*?\)".format(
model=model
),
source_code,
re.MULTILINE | re.DOTALL,
)
is not None
)
if not has_same_model_name:
issues.append(
{
"code": "RUNPYTHON_MODEL_VARIABLE_NAME",
"msg": (
"'{}': Model variable name {} is different from the "
"model class name that was found in the "
"apps.get_model(...) call."
).format(function_name, model),
}
)
return issues
def lint_runsql(self, runsql):
error = []
ignored = []
warning = []
# Detect warning on missing reverse operation
if not runsql.reversible:
issue = {
"code": "RUNSQL_REVERSIBLE",
"msg": "RunSQL data migration is not reversible",
}
if issue["code"] in self.exclude_migration_tests:
ignored.append(issue)
else:
warning.append(issue)
# Put the SQL in our SQL analyser
if runsql.sql != RunSQL.noop:
sql_statements = []
if isinstance(runsql.sql, (list, tuple)):
for sql in runsql.sql:
params = None
if isinstance(sql, (list, tuple)):
elements = len(sql)
if elements == 2:
sql, params = sql
else:
raise ValueError("Expected a 2-tuple but got %d" % elements)
sql_statements.append(sql % params)
else:
sql_statements.append(sql)
else:
sql_statements.append(runsql.sql)
sql_errors, sql_ignored, sql_warnings = analyse_sql_statements(
sql_statements,
settings.DATABASES[self.database]["ENGINE"],
self.exclude_migration_tests,
)
if sql_errors:
error += sql_errors
if sql_ignored:
ignored += sql_ignored
if sql_warnings:
warning += sql_warnings
# And analysse the reverse SQL
if runsql.reversible and runsql.reverse_sql != RunSQL.noop:
sql_statements = []
if isinstance(runsql.reverse_sql, (list, tuple)):
for sql in runsql.reverse_sql:
params = None
if isinstance(sql, (list, tuple)):
elements = len(sql)
if elements == 2:
sql, params = sql
else:
raise ValueError("Expected a 2-tuple but got %d" % elements)
sql_statements.append(sql % params)
else:
sql_statements.append(sql)
else:
sql_statements.append(runsql.reverse_sql)
sql_errors, sql_ignored, sql_warnings = analyse_sql_statements(
sql_statements,
settings.DATABASES[self.database]["ENGINE"],
self.exclude_migration_tests,
)
if sql_errors:
error += sql_errors
if sql_ignored:
ignored += sql_ignored
if sql_warnings:
warning += sql_warnings
return error, ignored, warning
