def tasks(self):
"""A list of all tasks that will be performed.
This can only be accessed after all necessary tasks have been queued.
"""
# If a caller is interested in the list of tasks, then it's likely
# interested in state on those tasks. That means we'll need to prepare
# all the tasks before we can return any of them.
self._prepare_tasks()
return six.itervalues(self._tasks_by_id)
def get_leaf_nodes(self):
"""Return all leaf nodes on the graph.
Leaf nodes are nodes that nothing depends on. These are generally the
last evolutions/migrations in any branch of the tree to apply.
Returns:
list of Node:
The list of leaf nodes, sorted by their insertion index.
"""
assert self._finalized
return sorted([
node
for node in six.itervalues(self._nodes) if not node.required_by
],
key=lambda node: node.insert_index)
def iter_indexes(self, table_name):
"""Iterate through all indexes for a table.
Args:
table_name (unicode):
The name of the table.
Yields:
IndexState:
An index in the table.
"""
table_name = self._norm_table_name(table_name)
for unique in (False, True):
try:
indexes = self._get_indexes_dict(table_name=table_name,
unique=unique)
except KeyError:
continue
for index_state in six.itervalues(indexes):
yield index_state
def to_sql(self):
"""Return a list of SQL statements for the table rebuild.
Any :py:attr:`alter_table` operations will be collapsed together into
a single table rebuild.
Returns:
list of unicode:
The list of SQL statements to run for the rebuild.
"""
evolver = self.evolver
model = self.model
connection = evolver.connection
qn = connection.ops.quote_name
table_name = model._meta.db_table
# Calculate some state for the rebuild operations, based on the
# Alter Table ops that were provided.
added_fields = []
deleted_columns = set()
renamed_columns = {}
replaced_fields = {}
added_constraints = []
new_initial = {}
reffed_renamed_cols = []
added_field_db_indexes = []
dropped_field_db_indexes = []
needs_rebuild = False
sql = []
for item in self.alter_table:
op = item['op']
if op == 'ADD COLUMN':
needs_rebuild = True
field = item['field']
if field.db_type(connection=connection) is not None:
initial = item['initial']
added_fields.append(field)
if initial is not None:
new_initial[field.column] = initial
elif op == 'DELETE COLUMN':
needs_rebuild = True
deleted_columns.add(item['column'])
elif op == 'RENAME COLUMN':
needs_rebuild = True
old_field = item['old_field']
new_field = item['new_field']
old_column = old_field.column
new_column = new_field.column
renamed_columns[old_column] = new_field.column
replaced_fields[old_column] = new_field
if evolver.is_column_referenced(table_name, old_column):
reffed_renamed_cols.append((old_column, new_column))
elif op == 'MODIFY COLUMN':
needs_rebuild = True
field = item['field']
initial = item['initial']
replaced_fields[field.column] = field
if initial is not None:
new_initial[field.column] = initial
elif op == 'CHANGE COLUMN TYPE':
needs_rebuild = True
old_field = item['old_field']
new_field = item['new_field']
column = old_field.column
replaced_fields[column] = new_field
elif op == 'ADD CONSTRAINTS':
needs_rebuild = True
added_constraints = item['constraints']
elif op == 'REBUILD':
# We're just rebuilding, not changing anything about it.
# This is used to get rid of auto-indexes from SQLite.
needs_rebuild = True
elif op == 'ADD DB INDEX':
added_field_db_indexes.append(item['field'])
elif op == 'DROP DB INDEX':
dropped_field_db_indexes.append(item['field'])
else:
raise ValueError(
'%s is not a valid Alter Table op for SQLite' % op)
for field in dropped_field_db_indexes:
sql += self.normalize_sql(evolver.drop_index(model, field))
if not needs_rebuild:
# We don't have any operations requiring a full table rebuild.
# We may have indexes to add (which would normally be added
# along with the rebuild).
for field in added_field_db_indexes:
sql += self.normalize_sql(evolver.create_index(model, field))
return self.pre_sql + self.sql + sql + self.post_sql
# Remove any Generic Fields.
old_fields = [
_field for _field in model._meta.local_fields
if _field.db_type(connection=connection) is not None
]
new_fields = [
replaced_fields.get(_field.column, _field)
for _field in old_fields + added_fields
if _field.column not in deleted_columns
]
field_values = OrderedDict()
for field in old_fields:
old_column = field.column
if old_column not in deleted_columns:
new_column = renamed_columns.get(old_column, old_column)
field_values[new_column] = qn(old_column)
field_initials = []
# If we have any new fields, add their defaults.
if new_initial:
for column, initial in six.iteritems(new_initial):
# Note that initial will only be None if null=True. Otherwise,
# it will be set to a user-defined callable or the default
# AddFieldInitialCallback, which will raise an exception in
# common code before we get too much further.
if initial is not None:
initial, embed_initial = evolver.normalize_initial(initial)
if embed_initial:
field_values[column] = initial
else:
field_initials.append(initial)
if column in field_values:
field_values[column] = \
'coalesce(%s, %%s)' % qn(column)
else:
field_values[column] = '%s'
# The SQLite documentation defines the steps that should be taken to
# safely alter the schema for a table. Unlike most types of databases,
# SQLite doesn't provide a general ALTER TABLE that can modify any
# part of the table, so for most things, we require a full table
# rebuild, and it must be done correctly.
#
# Step 1: Create a temporary table representing the new table
# schema. This will be temporary, and we don't need to worry
# about any indexes yet. Later, this will become the new
# table.
columns_sql = []
columns_sql_params = []
for field in new_fields:
if not isinstance(field, models.ManyToManyField):
schema = evolver.build_column_schema(model=model, field=field)
columns_sql.append('%s %s %s' %
(qn(schema['name']), schema['db_type'],
' '.join(schema['definition'])))
columns_sql_params += schema['definition_sql_params']
constraints_sql = []
if added_constraints:
# Django >= 2.2
with connection.schema_editor(collect_sql=True) as schema_editor:
for constraint in added_constraints:
constraint_sql = constraint.constraint_sql(
model, schema_editor)
if constraint_sql:
constraints_sql.append(constraint_sql)
sql.append((
'CREATE TABLE %s (%s);' %
(qn(TEMP_TABLE_NAME), ', '.join(columns_sql + constraints_sql)),
tuple(columns_sql_params),
))
# Step 2: Copy over any data from the old table into the new one.
sql.append(('INSERT INTO %s (%s) SELECT %s FROM %s;' % (
qn(TEMP_TABLE_NAME),
', '.join(qn(column) for column in six.iterkeys(field_values)),
', '.join(
six.text_type(_value)
for _value in six.itervalues(field_values)),
qn(table_name),
), tuple(field_initials)))
# Step 3: Drop the old table, making room for us to recreate the
# new schema table in its place.
sql += evolver.delete_table(table_name).to_sql()
# Step 4: Move over the temp table to the destination table name.
sql += evolver.rename_table(model=model,
old_db_table=TEMP_TABLE_NAME,
new_db_table=table_name).to_sql()
# Step 5: Restore any indexes.
class _Model(object):
class _meta(object):
db_table = table_name
local_fields = new_fields
db_tablespace = None
managed = True
proxy = False
swapped = False
index_together = []
indexes = []
sql += sql_indexes_for_model(connection, _Model)
# We've added all the indexes above. Any that were already there
# will be in the database state. However, if we've *specifically*
# had requests to add indexes, those ones won't be. We'll need to
# add them now.
#
# The easiest way is to use the same SQL generation functions we'd
# normally use to generate per-field indexes, since those track
# database state. We won't actually use the SQL.
for field in added_field_db_indexes:
evolver.create_index(model, field)
if reffed_renamed_cols:
# One or more tables referenced one or more renamed columns on
# this table, so now we need to update them.
#
# There are issues with renaming columns referenced by a foreign
# key in SQLite. Historically, we've allowed it, but the reality
# is that it can result in those foreign keys pointing to the
# wrong (old) column, causing any foreign key reference checks to
# fail. This is noticeable with Django 2.2+, which explicitly
# checks in its schema editor (which we invoke).
#
# We don't actually want or need to do a table rebuild on these.
# SQLite has another trick (and this is recommended in their
# documentation). We want to go through each of the tables that
# reference these columns and rewrite their table creation SQL
# in the sqlite_master table, and then tell SQLite to apply the
# new schema.
#
# This requires that we enable writable schemas and bump up the
# SQLite schema version for this database. This must be done at
# the moment we want to run this SQL statement, so we'll be
# adding this as a dynamic function to run later, rather than
# hard-coding any SQL now.
#
# Most of this can be done in a transaction, but not all. We have
# to execute much of this in its own transaction, and then write
# the new schema to disk with a VACUUM outside of a transaction.
def _update_refs(cursor):
schema_version = \
cursor.execute('PRAGMA schema_version').fetchone()[0]
refs_template = ' REFERENCES "%s" ("%%s") ' % table_name
return [
NewTransactionSQL([
# Allow us to update the database schema by
# manipulating the sqlite_master table.
'PRAGMA writable_schema = 1;',
] + [
# Update all tables that reference any renamed
# columns, setting their references to point to
# the new names.
('UPDATE sqlite_master SET sql ='
' replace(sql, %s, %s);',
(refs_template % old_column,
refs_template % new_column))
for old_column, new_column in reffed_renamed_cols
] + [
# Tell SQLite that we're done writing the schema,
# and give it a new schema version number.
('PRAGMA schema_version = %s;' % (schema_version + 1)),
'PRAGMA writable_schema = 0;',
# Make sure everything went well. We want to bail
# here before we commit the transaction if
# anything goes wrong.
'PRAGMA integrity_check;',
]),
NoTransactionSQL(['VACUUM;']),
]
sql.append(_update_refs)
return self.pre_sql + sql + self.sql + self.post_sql
def local_many_to_many(self):
"""A list of all local Many-to-Many fields on the model."""
return list(six.itervalues(self._many_to_many))
def local_fields(self):
"""A list of all local fields on the model."""
return list(six.itervalues(self._fields))