def assertQEqual(self, q1, q2, msg=None):
"""Assert that two Q objects are identical.
This will compare correctly for all supported versions of Django.
Version Added:
2.2
Args:
q1 (django.db.models.Q):
The first Q object.
q2 (django.db.models.Q):
The second Q object.
msg (unicode, optional):
An optional message to show if assertion fails.
Raises:
AssertionError:
The two Q objects were not equal.
"""
if django.VERSION[0] >= 2:
# Django 2.0+ supports equality checks for Q objects.
self._baseAssertEqual(q1, q2, msg=msg)
else:
# Django 1.11 and older does not, so we'll need to compare
# string representations.
#
# Note that this assumes that two Q() objects were constructed
# identically (for instance, both use native strings for field
# names, and not Unicode strings).
self.assertIsInstance(q1, Q, msg=msg)
self.assertIsInstance(q2, Q, msg=msg)
self.assertEqual(six.text_type(q1), six.text_type(q2), msg=msg)
def assertFEqual(self, f1, f2):
"""Assert that two F objects are identical.
This will compare correctly for all supported versions of Django.
Version Added:
2.2
Args:
f1 (django.db.models.F):
The first F object.
f2 (django.db.models.F):
The second F object.
Raises:
AssertionError:
The two F objects were not equal.
"""
if django.VERSION[0] >= 2:
# Django 2.0+ supports equality checks for F objects.
self._baseAssertEqual(f1, f2)
else:
# Django 1.11 and older does not, so we'll need to compare
# string representations.
#
# Note that this assumes that two F() objects were constructed
# identically (for instance, both use native strings for field
# names, and not Unicode strings).
self.assertIsInstance(f1, F)
self.assertIsInstance(f2, F)
self.assertEqual(six.text_type(f1), six.text_type(f2))
def execute(self, cursor=None, sql_executor=None, **kwargs):
"""Execute the task.
This will delete any tables owned by the application.
Args:
cursor (django.db.backends.util.CursorWrapper, unused):
The legacy database cursor. This is no longer used.
sql_executor (django_evolution.utils.sql.SQLExecutor, optional):
The SQL executor used to run any SQL on the database.
Raises:
django_evolution.errors.EvolutionExecutionError:
The evolution task failed. Details are in the error.
"""
assert sql_executor
if self.evolution_required:
try:
sql_executor.run_sql(self.sql, execute=True)
except Exception as e:
raise EvolutionExecutionError(
_('Error purging app "%s": %s')
% (self.app_label, e),
app_label=self.app_label,
detailed_error=six.text_type(e),
last_sql_statement=getattr(e, 'last_sql_statement'))
def run_checks(self):
"""Perform checks on the migrations and any history.
Raises:
django_evolution.errors.MigrationConflictsError:
There are conflicts between migrations loaded from disk.
django_evolution.errors.MigrationHistoryError:
There are unapplied dependencies to applied migrations.
"""
# Make sure that the migration files in the tree form a proper history.
if hasattr(self.loader, 'check_consistent_history'):
# Django >= 1.10
from django.db.migrations.exceptions import \
InconsistentMigrationHistory
try:
self.loader.check_consistent_history(self.connection)
except InconsistentMigrationHistory as e:
raise MigrationHistoryError(six.text_type(e))
# Now check that there aren't any conflicts between any migrations that
# we may end up working with.
conflicts = self.loader.detect_conflicts()
if conflicts:
raise MigrationConflictsError(conflicts)
def add_argument(self, *args, **kwargs):
"""Add an argument to the parser.
This is a simple wrapper that provides compatibility with most of
:py:meth:`argparse.ArgumentParser.add_argument`. It supports the
types that :py:meth:`optparse.OptionParser.add_option` supports (though
those types should be passed as the primitive types and not as the
string names).
Args:
*args (tuple):
Positional arguments to pass to
:py:meth:`optparse.OptionParser.add_option`.
**kwargs (dict):
Keyword arguments to pass to
:py:meth:`optparse.OptionParser.add_option`.
"""
if not args[0].startswith('-'):
# This is a positional argument, which is not supported by
# optparse.
return
arg_type = kwargs.get('type')
if arg_type is not None:
kwargs['type'] = six.text_type(arg_type.__name__)
self.parser.add_option(*args, **kwargs)
def _save_project_sig(self, new_evolutions):
"""Save the project signature and any new evolutions.
This will serialize the current modified project signature to the
database and write any new evolutions, attaching them to the current
project version.
This can be called many times for one evolver instance. After the
first time, the version already saved will simply be updated.
Args:
new_evolutions (list of django_evolution.models.Evolution):
The list of new evolutions to save to the database.
Raises:
django_evolution.errors.EvolutionExecutionError:
There was an error saving to the database.
"""
version = self.version
if version is None:
version = Version(signature=self.project_sig)
self.version = version
try:
version.save(using=self.database_name)
if new_evolutions:
for evolution in new_evolutions:
evolution.version = version
Evolution.objects.using(
self.database_name).bulk_create(new_evolutions)
except Exception as e:
raise EvolutionExecutionError(
_('Error saving new evolution version information: %s') % e,
detailed_error=six.text_type(e))
def handle(self, *app_labels, **options):
"""Handle the command.
This will validate the arguments and run through the evolution
process.
Args:
app_labels (list of unicode):
The app labels to evolve.
options (dict):
Options parsed by the argument parser.
Raises:
django.core.management.base.CommandError:
Arguments were invalid or something went wrong. Details are
in the message.
"""
if not django_evolution_settings.ENABLED:
raise CommandError(
_('Django Evolution is disabled for this project. '
'Evolutions cannot be manually run.'))
self.purge = options['purge']
self.verbosity = int(options['verbosity'])
hint = options['hint']
compile_sql = options['compile_sql']
database_name = options['database'] or DEFAULT_DB_ALIAS
execute = options['execute']
interactive = options['interactive']
write_evolution_name = options['write_evolution_name']
if app_labels and self.execute:
raise CommandError(
_('Cannot specify an application name when executing '
'evolutions.'))
if write_evolution_name and not hint:
raise CommandError(_('--write cannot be used without --hint.'))
import_management_modules()
try:
self.evolver = Evolver(database_name=database_name,
hinted=hint,
verbosity=self.verbosity,
interactive=interactive)
# Figure out what tasks we need to add to the evolver. This
# must be done before we check any state (as that will finalize
# the task list).
self._add_tasks(app_labels)
# Calculate some information we may need later.
self.active_purge_tasks = [
task for task in self.evolver.tasks
if isinstance(task, PurgeAppTask) and len(task.sql) > 0
]
# Display any additional information on the evolution process
# the caller may be interested in.
if self.verbosity > 1:
self._display_extra_task_details()
# Simulate the evolutions to make sure that they'll get us to the
# target database state. This will raise a CommandError with
# helpful information if the evolutions don't get us there, or
# if one or more evolutions couldn't be simulated.
simulated = self._check_simulation()
if not self.evolver.get_evolution_required():
if self.verbosity > 0:
self.stdout.write(_('No database upgrade required.\n'))
elif execute:
if not interactive or self._confirm_execute():
self._perform_evolution()
else:
self.stderr.write(_('Database upgrade cancelled.\n'))
elif compile_sql:
self._display_compiled_sql()
else:
# Be helpful and list any applications that can be purged,
# and then show any evolution content that may be useful to
# the user.
self._display_available_purges()
self._generate_evolution_contents(write_evolution_name)
if simulated:
if self.verbosity > 0:
self.stdout.write(_('Trial upgrade successful!\n'))
if not self.evolver.hinted and self.verbosity > 0:
self.stdout.write(
_('Run `./manage.py evolve --execute` to apply '
'the evolution.\n'))
except EvolutionException as e:
raise CommandError(six.text_type(e))
def _perform_evolution(self):
"""Perform the evolution.
This will perform the evolution, based on the options passed to this
command. Progress on the evolution will be printed to the console.
Raises:
django.core.management.base.CommandError:
The evolution failed.
"""
evolver = self.evolver
verbosity = self.verbosity
if verbosity > 0:
@receiver(applying_evolution, sender=evolver)
def _on_applying_evolution(task, evolutions, **kwargs):
if verbosity > 2:
message = (
_('Applying database evolutions for %(app_label)s '
'(%(evolution_labels)s)...\n') % {
'app_label':
task.app_label,
'evolution_labels':
', '.join(evolution.label
for evolution in evolutions),
})
else:
message = (_('Applying database evolutions for '
'%(app_label)s...\n') % {
'app_label': task.app_label,
})
self.stdout.write(message)
@receiver(applying_migration, sender=evolver)
def _on_applying_migration(migration, **kwargs):
self.stdout.write(
_('Applying database migration %(migration_name)s for '
'%(app_label)s...\n') % {
'app_label': migration.app_label,
'migration_name': migration.name,
})
@receiver(creating_models, sender=evolver)
def _on_creating_models(app_label, model_names, **kwargs):
if verbosity > 2:
message = (
_('Creating new database models for %(app_label)s '
'(%(model_names)s)...\n') % {
'app_label': app_label,
'model_names': ', '.join(model_names),
})
else:
message = (_('Creating new database models for '
'%(app_label)s...\n') % {
'app_label': app_label,
})
self.stdout.write(message)
if verbosity > 1:
@receiver(applied_evolution, sender=evolver)
def _on_applied_evolution(task, evolutions, **kwargs):
if verbosity > 2:
message = (
_('Successfully applied database evolutions for '
'%(app_label)s (%(evolution_labels)s).\n') % {
'app_label':
task.app_label,
'evolution_labels':
', '.join(evolution.label
for evolution in evolutions),
})
else:
message = (
_('Successfully applied database evolutions for '
'%(app_label)s.\n') % {
'app_label': task.app_label,
})
self.stdout.write(message)
@receiver(applied_migration, sender=evolver)
def _on_applied_migration(migration, **kwargs):
self.stdout.write(
_('Successfully applied database migration '
'%(migration_name)s for %(app_label)s.\n') % {
'app_label': migration.app_label,
'migration_name': migration.name,
})
@receiver(created_models, sender=evolver)
def _on_created_models(app_label, model_names, **kwargs):
if verbosity > 2:
message = (
_('Successfully created new database models for '
'%(app_label)s (%(model_names)s).\n') % {
'app_label': app_label,
'model_names': ', '.join(model_names),
})
else:
message = (
_('Successfully created new database models for '
'%(app_label)s.\n') % {
'app_label': app_label,
})
self.stdout.write(message)
self.stdout.write('\n%s\n\n' % self._wrap_paragraphs(
_('This may take a while. Please be patient, and DO NOT '
'cancel the upgrade!')))
try:
evolver.evolve()
except EvolutionException as e:
self.stderr.write('%s\n' % e)
if getattr(e, 'last_sql_statement', None):
self.stderr.write(
_('The SQL statement that failed was: %s\n') %
(e.last_sql_statement, ))
raise CommandError(six.text_type(e))
if verbosity > 0:
if evolver.installed_new_database:
self.stdout.write(_('The database creation was successful!\n'))
else:
self.stdout.write(_('The database upgrade was successful!\n'))
def to_sql(self):
"""Return a list of SQL statements for the table rebuild.
Any :py:attr:`alter_table` operations will be collapsed together into
a single table rebuild.
Returns:
list of unicode:
The list of SQL statements to run for the rebuild.
"""
evolver = self.evolver
model = self.model
connection = evolver.connection
qn = connection.ops.quote_name
table_name = model._meta.db_table
# Calculate some state for the rebuild operations, based on the
# Alter Table ops that were provided.
added_fields = []
deleted_columns = set()
renamed_columns = {}
replaced_fields = {}
added_constraints = []
new_initial = {}
reffed_renamed_cols = []
added_field_db_indexes = []
dropped_field_db_indexes = []
needs_rebuild = False
sql = []
for item in self.alter_table:
op = item['op']
if op == 'ADD COLUMN':
needs_rebuild = True
field = item['field']
if field.db_type(connection=connection) is not None:
initial = item['initial']
added_fields.append(field)
if initial is not None:
new_initial[field.column] = initial
elif op == 'DELETE COLUMN':
needs_rebuild = True
deleted_columns.add(item['column'])
elif op == 'RENAME COLUMN':
needs_rebuild = True
old_field = item['old_field']
new_field = item['new_field']
old_column = old_field.column
new_column = new_field.column
renamed_columns[old_column] = new_field.column
replaced_fields[old_column] = new_field
if evolver.is_column_referenced(table_name, old_column):
reffed_renamed_cols.append((old_column, new_column))
elif op == 'MODIFY COLUMN':
needs_rebuild = True
field = item['field']
initial = item['initial']
replaced_fields[field.column] = field
if initial is not None:
new_initial[field.column] = initial
elif op == 'CHANGE COLUMN TYPE':
needs_rebuild = True
old_field = item['old_field']
new_field = item['new_field']
column = old_field.column
replaced_fields[column] = new_field
elif op == 'ADD CONSTRAINTS':
needs_rebuild = True
added_constraints = item['constraints']
elif op == 'REBUILD':
# We're just rebuilding, not changing anything about it.
# This is used to get rid of auto-indexes from SQLite.
needs_rebuild = True
elif op == 'ADD DB INDEX':
added_field_db_indexes.append(item['field'])
elif op == 'DROP DB INDEX':
dropped_field_db_indexes.append(item['field'])
else:
raise ValueError(
'%s is not a valid Alter Table op for SQLite' % op)
for field in dropped_field_db_indexes:
sql += self.normalize_sql(evolver.drop_index(model, field))
if not needs_rebuild:
# We don't have any operations requiring a full table rebuild.
# We may have indexes to add (which would normally be added
# along with the rebuild).
for field in added_field_db_indexes:
sql += self.normalize_sql(evolver.create_index(model, field))
return self.pre_sql + self.sql + sql + self.post_sql
# Remove any Generic Fields.
old_fields = [
_field for _field in model._meta.local_fields
if _field.db_type(connection=connection) is not None
]
new_fields = [
replaced_fields.get(_field.column, _field)
for _field in old_fields + added_fields
if _field.column not in deleted_columns
]
field_values = OrderedDict()
for field in old_fields:
old_column = field.column
if old_column not in deleted_columns:
new_column = renamed_columns.get(old_column, old_column)
field_values[new_column] = qn(old_column)
field_initials = []
# If we have any new fields, add their defaults.
if new_initial:
for column, initial in six.iteritems(new_initial):
# Note that initial will only be None if null=True. Otherwise,
# it will be set to a user-defined callable or the default
# AddFieldInitialCallback, which will raise an exception in
# common code before we get too much further.
if initial is not None:
initial, embed_initial = evolver.normalize_initial(initial)
if embed_initial:
field_values[column] = initial
else:
field_initials.append(initial)
if column in field_values:
field_values[column] = \
'coalesce(%s, %%s)' % qn(column)
else:
field_values[column] = '%s'
# The SQLite documentation defines the steps that should be taken to
# safely alter the schema for a table. Unlike most types of databases,
# SQLite doesn't provide a general ALTER TABLE that can modify any
# part of the table, so for most things, we require a full table
# rebuild, and it must be done correctly.
#
# Step 1: Create a temporary table representing the new table
# schema. This will be temporary, and we don't need to worry
# about any indexes yet. Later, this will become the new
# table.
columns_sql = []
columns_sql_params = []
for field in new_fields:
if not isinstance(field, models.ManyToManyField):
schema = evolver.build_column_schema(model=model, field=field)
columns_sql.append('%s %s %s' %
(qn(schema['name']), schema['db_type'],
' '.join(schema['definition'])))
columns_sql_params += schema['definition_sql_params']
constraints_sql = []
if added_constraints:
# Django >= 2.2
with connection.schema_editor(collect_sql=True) as schema_editor:
for constraint in added_constraints:
constraint_sql = constraint.constraint_sql(
model, schema_editor)
if constraint_sql:
constraints_sql.append(constraint_sql)
sql.append((
'CREATE TABLE %s (%s);' %
(qn(TEMP_TABLE_NAME), ', '.join(columns_sql + constraints_sql)),
tuple(columns_sql_params),
))
# Step 2: Copy over any data from the old table into the new one.
sql.append(('INSERT INTO %s (%s) SELECT %s FROM %s;' % (
qn(TEMP_TABLE_NAME),
', '.join(qn(column) for column in six.iterkeys(field_values)),
', '.join(
six.text_type(_value)
for _value in six.itervalues(field_values)),
qn(table_name),
), tuple(field_initials)))
# Step 3: Drop the old table, making room for us to recreate the
# new schema table in its place.
sql += evolver.delete_table(table_name).to_sql()
# Step 4: Move over the temp table to the destination table name.
sql += evolver.rename_table(model=model,
old_db_table=TEMP_TABLE_NAME,
new_db_table=table_name).to_sql()
# Step 5: Restore any indexes.
class _Model(object):
class _meta(object):
db_table = table_name
local_fields = new_fields
db_tablespace = None
managed = True
proxy = False
swapped = False
index_together = []
indexes = []
sql += sql_indexes_for_model(connection, _Model)
# We've added all the indexes above. Any that were already there
# will be in the database state. However, if we've *specifically*
# had requests to add indexes, those ones won't be. We'll need to
# add them now.
#
# The easiest way is to use the same SQL generation functions we'd
# normally use to generate per-field indexes, since those track
# database state. We won't actually use the SQL.
for field in added_field_db_indexes:
evolver.create_index(model, field)
if reffed_renamed_cols:
# One or more tables referenced one or more renamed columns on
# this table, so now we need to update them.
#
# There are issues with renaming columns referenced by a foreign
# key in SQLite. Historically, we've allowed it, but the reality
# is that it can result in those foreign keys pointing to the
# wrong (old) column, causing any foreign key reference checks to
# fail. This is noticeable with Django 2.2+, which explicitly
# checks in its schema editor (which we invoke).
#
# We don't actually want or need to do a table rebuild on these.
# SQLite has another trick (and this is recommended in their
# documentation). We want to go through each of the tables that
# reference these columns and rewrite their table creation SQL
# in the sqlite_master table, and then tell SQLite to apply the
# new schema.
#
# This requires that we enable writable schemas and bump up the
# SQLite schema version for this database. This must be done at
# the moment we want to run this SQL statement, so we'll be
# adding this as a dynamic function to run later, rather than
# hard-coding any SQL now.
#
# Most of this can be done in a transaction, but not all. We have
# to execute much of this in its own transaction, and then write
# the new schema to disk with a VACUUM outside of a transaction.
def _update_refs(cursor):
schema_version = \
cursor.execute('PRAGMA schema_version').fetchone()[0]
refs_template = ' REFERENCES "%s" ("%%s") ' % table_name
return [
NewTransactionSQL([
# Allow us to update the database schema by
# manipulating the sqlite_master table.
'PRAGMA writable_schema = 1;',
] + [
# Update all tables that reference any renamed
# columns, setting their references to point to
# the new names.
('UPDATE sqlite_master SET sql ='
' replace(sql, %s, %s);',
(refs_template % old_column,
refs_template % new_column))
for old_column, new_column in reffed_renamed_cols
] + [
# Tell SQLite that we're done writing the schema,
# and give it a new schema version number.
('PRAGMA schema_version = %s;' % (schema_version + 1)),
'PRAGMA writable_schema = 0;',
# Make sure everything went well. We want to bail
# here before we commit the transaction if
# anything goes wrong.
'PRAGMA integrity_check;',
]),
NoTransactionSQL(['VACUUM;']),
]
sql.append(_update_refs)
return self.pre_sql + sql + self.sql + self.post_sql