def sequence_list(self):
"""
Return a list of information about all DB sequences for all models in
all apps.
"""
from django.apps import apps
from django.db import router
sequence_list = []
cursor = self.connection.cursor()
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config, self.connection.alias):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
sequence_list.extend(self.get_sequences(cursor, model._meta.db_table, model._meta.local_fields))
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.remote_field.through is None:
sequence = self.get_sequences(cursor, f.m2m_db_table())
sequence_list.extend(sequence or [{'table': f.m2m_db_table(), 'column': None}])
return sequence_list
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
created_models = set()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False))
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not (
(converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)
)
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=connection.features.can_rollback_ddl):
deferred_sql = []
for app_name, model_list in manifest.items():
for model in model_list:
if not model._meta.can_migrate(connection):
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" % (app_name, model._meta.object_name)
)
with connection.schema_editor() as editor:
if self.verbosity >= 1:
self.stdout.write(" Creating table %s\n" % model._meta.db_table)
editor.create_model(model)
deferred_sql.extend(editor.deferred_sql)
editor.deferred_sql = []
created_models.add(model)
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL...\n")
for statement in deferred_sql:
cursor.execute(statement)
finally:
cursor.close()
return created_models
def sequence_list(self):
"Returns a list of information about all DB sequences for all models in all apps."
from django.apps import apps
from django.db import models, router
sequence_list = []
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config, self.connection.alias):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
for f in model._meta.local_fields:
if isinstance(f, models.AutoField):
sequence_list.append({'table': model._meta.db_table, 'column': f.column})
break # Only one AutoField is allowed per model, so don't bother continuing.
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.remote_field.through is None:
sequence_list.append({'table': f.m2m_db_table(), 'column': None})
return sequence_list
def sequence_list(self):
"""
Return a list of information about all DB sequences for all models in
all apps.
"""
from django.apps import apps
from django.db import router
sequence_list = []
with self.connection.cursor() as cursor:
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(
app_config, self.connection.alias):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
sequence_list.extend(
self.get_sequences(cursor, model._meta.db_table,
model._meta.local_fields))
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.remote_field.through._meta.auto_created:
sequence = self.get_sequences(
cursor, f.m2m_db_table())
sequence_list.extend(sequence
or [{
'table': f.m2m_db_table(),
'column': None
}])
return sequence_list
def django_table_names(self, only_existing=False, include_views=True):
"""
Returns a list of all table names that have associated Django models and
are in INSTALLED_APPS.
If only_existing is True, the resulting list will only include the tables
that actually exist in the database.
"""
from django.apps import apps
from django.db import router
tables = set()
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config, self.connection.alias):
if not model._meta.managed:
continue
tables.add(model._meta.db_table)
tables.update(f.m2m_db_table() for f in model._meta.local_many_to_many)
tables = list(tables)
if only_existing:
existing_tables = self.table_names(include_views=include_views)
tables = [
t
for t in tables
if self.table_name_converter(t) in existing_tables
]
return tables
def sync_apps(self, connection, app_labels):
"""Run the old syncdb-style operation on a list of app_labels."""
with connection.cursor() as cursor:
tables = connection.introspection.table_names(cursor)
# Build the manifest of apps and models that are to be synchronized.
all_models = [
(
app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False),
)
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.identifier_converter
return not (
(converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)
)
manifest = {
app_name: list(filter(model_installed, model_list))
for app_name, model_list in all_models
}
# Create the tables for each model
if self.verbosity >= 1:
self.stdouimport time
def get_migratable_models(self):
from django.apps import apps
from django.db import router
return (model for app_config in apps.get_app_configs()
for model in router.get_migratable_models(
app_config, self.connection.alias)
if model._meta.can_migrate(self.connection))
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
created_models = set()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False))
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not (
(converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)
)
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=connection.features.can_rollback_ddl):
deferred_sql = []
for app_name, model_list in manifest.items():
for model in model_list:
if not model._meta.can_migrate(connection):
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" % (app_name, model._meta.object_name)
)
with connection.schema_editor() as editor:
if self.verbosity >= 1:
self.stdout.write(" Creating table %s\n" % model._meta.db_table)
editor.create_model(model)
deferred_sql.extend(editor.deferred_sql)
editor.deferred_sql = []
created_models.add(model)
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL...\n")
for statement in deferred_sql:
cursor.execute(statement)
finally:
cursor.close()
return created_models
def sql_create(app_config, style, connection):
"Returns a list of the CREATE TABLE SQL statements for the given app."
if connection.settings_dict['ENGINE'] == 'django.db.backends.dummy':
# This must be the "dummy" database backend, which means the user
# hasn't set ENGINE for the database.
raise CommandError(
"Django doesn't know which syntax to use for your SQL statements,"
"\nbecause you haven't properly specified the ENGINE setting for "
"the database.\n"
"see: https://docs.djangoproject.com/en/%s/ref/settings/"
"#databases" % get_docs_version())
# Get installed models, so we generate REFERENCES right.
# We trim models from the current app so that the sqlreset command does not
# generate invalid SQL (leaving models out of known_models is harmless, so
# we can be conservative).
final_output = []
output = []
for model in router.get_migratable_models(app_config,
connection.alias,
include_auto_created=True):
with connection.schema_editor() as editor:
output += sql_create_model(editor, model)
final_output.extend(editor.deferred_sql)
editor.deferred_sql = []
if get_docs_version().startswith('2'):
# Content of final_output are Statement objects which need to be
# converted as str
final_output = [str(statement) for statement in final_output]
return output + final_output
def emit_post_migrate(verbosity, interactive, database):
# Emit the post migrate signal. This allows individual applications to
# respond as if the database had been migrated from scratch.
all_models = []
for app_config in apps.get_app_configs():
all_models.extend(router.get_migratable_models(app_config, database, include_auto_created=True))
emit_post_migrate_signal(set(all_models), verbosity, interactive, database)
def emit_post_migrate(verbosity, interactive, database):
# Emit the post migrate signal. This allows individual applications to
# respond as if the database had been migrated from scratch.
all_models = []
for app_config in apps.get_app_configs():
all_models.extend(router.get_migratable_models(app_config, database, include_auto_created=True))
emit_post_migrate_signal(set(all_models), verbosity, interactive, database)
def sequence_list(self):
"""
Return a list of information about all DB sequences for all models in
all apps.
"""
from django.apps import apps
from django.db import models, router
sequence_list = []
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config,
self.connection.alias):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
for f in model._meta.local_fields:
if isinstance(f, models.AutoField):
sequence_list.append({
'table': model._meta.db_table,
'column': f.column
})
break # Only one AutoField is allowed per model, so don't bother continuing.
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.remote_field.through is None:
sequence_list.append({
'table': f.m2m_db_table(),
'column': None
})
return sequence_list
def handle(self, *args, **options):
db_values = []
db = options.get('database')
connection = connections[db]
connection.prepare_database()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False))
for app_config in apps.get_app_configs()
]
for app_name, model_list in all_models:
for m in model_list:
if hasattr(m, '_translation_fields'):
for x in m._translation_fields:
for y in m.objects.all():
db_values.append(getattr(y, x))
path = apps.app_configs[app_name].path
# print db_values
with open(os.path.join(path, 'extra_translations.py'), 'w') as f:
translations_file = File(f)
translations_file.write('''# encoding: utf-8
from __future__ import unicode_literals, absolute_import
from django.utils.translation import ugettext_lazy as _
translations = (
''')
for v in db_values:
translations_file.write(""" _('%s'),\n""" % smart_str(v.replace('\'', '\\\'')))
translations_file.write(')')
def django_table_names(self, only_existing=False, include_views=True):
"""
Return a list of all table names that have associated Django models and
are in INSTALLED_APPS.
If only_existing is True, include only the tables in the database.
"""
from django.apps import apps
from django.db import router
tables = set()
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config,
self.connection.alias):
if not model._meta.managed:
continue
tables.add(model._meta.db_table)
tables.update(f.m2m_db_table()
for f in model._meta.local_many_to_many
if f.remote_field.through._meta.managed)
tables = list(tables)
if only_existing:
existing_tables = self.table_names(include_views=include_views)
tables = [
t for t in tables
if self.table_name_converter(t) in existing_tables
]
return tables
def sql_indexes(app_config, style, connection):
"Returns a list of the CREATE INDEX SQL statements for all models in the given app."
output = []
for model in router.get_migratable_models(app_config,
connection.alias,
include_auto_created=True):
output.extend(connection.creation.sql_indexes_for_model(model, style))
return output
def sql_destroy_indexes(app_config, style, connection):
"Returns a list of the DROP INDEX SQL statements for all models in the given app."
check_for_migrations(app_config, connection)
output = []
for model in router.get_migratable_models(app_config, connection.alias, include_auto_created=True):
output.extend(connection.creation.sql_destroy_indexes_for_model(model, style))
return output
def sync_apps(self, connection, app_labels):
"""Run the old syncdb-style operation on a list of app_labels."""
with connection.cursor() as cursor:
tables = connection.introspection.table_names(cursor)
# Build the manifest of apps and models that are to be synchronized.
all_models = [
(
app_config.label,
router.get_migratable_models(
app_config, connection.alias, include_auto_created=False
),
)
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.identifier_converter
return not (
(converter(opts.db_table) in tables)
or (
opts.auto_created
and converter(opts.auto_created._meta.db_table) in tables
)
)
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with connection.schema_editor() as editor:
for app_name, model_list in manifest.items():
for model in model_list:
# Never install unmanaged models, etc.
if not model._meta.can_migrate(connection):
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n"
% (app_name, model._meta.object_name)
)
if self.verbosity >= 1:
self.stdout.write(
" Creating table %s\n" % model._meta.db_table
)
editor.create_model(model)
# Deferred SQL is executed when exiting the editor's context.
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL...\n")
def sql_delete(app_config, style, connection, close_connection=True):
"Returns a list of the DROP TABLE SQL statements for the given app."
check_for_migrations(app_config, connection)
# This should work even if a connection isn't available
try:
cursor = connection.cursor()
except Exception:
cursor = None
try:
# Figure out which tables already exist
if cursor:
table_names = connection.introspection.table_names(cursor)
else:
table_names = []
output = []
# Output DROP TABLE statements for standard application tables.
to_delete = set()
references_to_delete = {}
app_models = router.get_migratable_models(app_config,
connection.alias,
include_auto_created=True)
for model in app_models:
if cursor and connection.introspection.table_name_converter(
model._meta.db_table) in table_names:
# The table exists, so it needs to be dropped
opts = model._meta
for f in opts.local_fields:
if f.rel and f.rel.to not in to_delete:
references_to_delete.setdefault(f.rel.to, []).append(
(model, f))
to_delete.add(model)
for model in app_models:
if connection.introspection.table_name_converter(
model._meta.db_table) in table_names:
output.extend(
connection.creation.sql_destroy_model(
model, references_to_delete, style))
finally:
# Close database connection explicitly, in case this output is being piped
# directly into a database client, to avoid locking issues.
if cursor and close_connection:
cursor.close()
connection.close()
if not output:
output.append(
'-- App creates no tables in the database. Nothing to do.')
return output[::-1] # Reverse it, to deal with table dependencies.
def sql_custom(app, style, connection):
"Returns a list of the custom table modifying SQL statements for the given app."
output = []
app_models = router.get_migratable_models(app, connection.alias)
for model in app_models:
output.extend(custom_sql_for_model(model, style, connection))
return output
def installed_models(self, tables):
"Returns a set of all models represented by the provided list of table names."
from django.apps import apps
from django.db import router
all_models = []
for app_config in apps.get_app_configs():
all_models.extend(router.get_migratable_models(app_config, self.connection.alias))
tables = list(map(self.table_name_converter, tables))
return {m for m in all_models if self.table_name_converter(m._meta.db_table) in tables}
def sql_custom(app_config, style, connection):
"Returns a list of the custom table modifying SQL statements for the given app."
output = []
app_models = router.get_migratable_models(app_config, connection.alias)
for model in app_models:
output.extend(custom_sql_for_model(model, style, connection))
return output
def installed_models(self, tables):
"Returns a set of all models represented by the provided list of table names."
from django.apps import apps
from django.db import router
all_models = []
for app_config in apps.get_app_configs():
all_models.extend(router.get_migratable_models(app_config, self.connection.alias))
tables = list(map(self.table_name_converter, tables))
return {
m for m in all_models
if self.table_name_converter(m._meta.db_table) in tables
}
def sql_create(app_config, style, connection):
"Returns a list of the CREATE TABLE SQL statements for the given app."
check_for_migrations(app_config, connection)
if connection.settings_dict["ENGINE"] == "django.db.backends.dummy":
# This must be the "dummy" database backend, which means the user
# hasn't set ENGINE for the database.
raise CommandError(
"Django doesn't know which syntax to use for your SQL statements,\n"
+ "because you haven't properly specified the ENGINE setting for the database.\n"
+ "see: https://docs.djangoproject.com/en/dev/ref/settings/#databases"
)
# Get installed models, so we generate REFERENCES right.
# We trim models from the current app so that the sqlreset command does not
# generate invalid SQL (leaving models out of known_models is harmless, so
# we can be conservative).
app_models = app_config.get_models(include_auto_created=True)
final_output = []
tables = connection.introspection.table_names()
known_models = set(model for model in connection.introspection.installed_models(tables) if model not in app_models)
pending_references = {}
for model in router.get_migratable_models(app_config, connection.alias, include_auto_created=True):
output, references = connection.creation.sql_create_model(model, style, known_models)
final_output.extend(output)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in known_models:
final_output.extend(connection.creation.sql_for_pending_references(refto, style, pending_references))
final_output.extend(connection.creation.sql_for_pending_references(model, style, pending_references))
# Keep track of the fact that we've created the table for this model.
known_models.add(model)
# Handle references to tables that are from other apps
# but don't exist physically.
not_installed_models = set(pending_references.keys())
if not_installed_models:
alter_sql = []
for model in not_installed_models:
alter_sql.extend(
[
"-- " + sql
for sql in connection.creation.sql_for_pending_references(model, style, pending_references)
]
)
if alter_sql:
final_output.append("-- The following references should be added but depend on non-existent tables:")
final_output.extend(alter_sql)
return final_output
def sql_delete(app_config, style, connection, close_connection=True):
"Returns a list of the DROP TABLE SQL statements for the given app."
check_for_migrations(app_config, connection)
# This should work even if a connection isn't available
try:
cursor = connection.cursor()
except Exception:
cursor = None
try:
# Figure out which tables already exist
if cursor:
table_names = connection.introspection.table_names(cursor)
else:
table_names = []
output = []
# Output DROP TABLE statements for standard application tables.
to_delete = set()
references_to_delete = {}
app_models = router.get_migratable_models(app_config, connection.alias, include_auto_created=True)
for model in app_models:
if cursor and connection.introspection.table_name_converter(model._meta.db_table) in table_names:
# The table exists, so it needs to be dropped
opts = model._meta
for f in opts.local_fields:
if f.rel and f.rel.to not in to_delete:
references_to_delete.setdefault(f.rel.to, []).append((model, f))
to_delete.add(model)
for model in app_models:
if connection.introspection.table_name_converter(model._meta.db_table) in table_names:
output.extend(connection.creation.sql_destroy_model(model, references_to_delete, style))
finally:
# Close database connection explicitly, in case this output is being piped
# directly into a database client, to avoid locking issues.
if cursor and close_connection:
cursor.close()
connection.close()
if not output:
output.append('-- App creates no tables in the database. Nothing to do.')
return output[::-1] # Reverse it, to deal with table dependencies.
def sync_apps(self, connection, app_labels):
"""Run the old syncdb-style operation on a list of app_labels."""
with connection.cursor() as cursor:
tables = connection.introspection.table_names(cursor)
# Build the manifest of apps and models that are to be synchronized.
all_models = [
(
app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False),
)
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
return not (
(converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)
)
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables…\n")
with connection.schema_editor() as editor:
for app_name, model_list in manifest.items():
for model in model_list:
# Never install unmanaged models, etc.
if not model._meta.can_migrate(connection):
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" % (app_name, model._meta.object_name)
)
if self.verbosity >= 1:
self.stdout.write(" Creating table %s\n" % model._meta.db_table)
editor.create_model(model)
# Deferred SQL is executed when exiting the editor's context.
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL…\n")
def sql_create(app_config, style, connection):
"Returns a list of the CREATE TABLE SQL statements for the given app."
check_for_migrations(app_config, connection)
if connection.settings_dict['ENGINE'] == 'django.db.backends.dummy':
# This must be the "dummy" database backend, which means the user
# hasn't set ENGINE for the database.
raise CommandError("Django doesn't know which syntax to use for your SQL statements,\n" +
"because you haven't properly specified the ENGINE setting for the database.\n" +
"see: https://docs.djangoproject.com/en/dev/ref/settings/#databases")
# Get installed models, so we generate REFERENCES right.
# We trim models from the current app so that the sqlreset command does not
# generate invalid SQL (leaving models out of known_models is harmless, so
# we can be conservative).
app_models = list(app_config.get_models(include_auto_created=True))
final_output = []
tables = connection.introspection.table_names()
known_models = set(model for model in connection.introspection.installed_models(tables) if model not in app_models)
pending_references = {}
for model in router.get_migratable_models(app_config, connection.alias, include_auto_created=True):
output, references = connection.creation.sql_create_model(model, style, known_models)
final_output.extend(output)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in known_models:
final_output.extend(connection.creation.sql_for_pending_references(refto, style, pending_references))
final_output.extend(connection.creation.sql_for_pending_references(model, style, pending_references))
# Keep track of the fact that we've created the table for this model.
known_models.add(model)
# Handle references to tables that are from other apps
# but don't exist physically.
not_installed_models = set(pending_references.keys())
if not_installed_models:
alter_sql = []
for model in not_installed_models:
alter_sql.extend(['-- ' + sql for sql in
connection.creation.sql_for_pending_references(model, style, pending_references)])
if alter_sql:
final_output.append('-- The following references should be added but depend on non-existent tables:')
final_output.extend(alter_sql)
return final_output
def sequencelist(self):
"""Returns a list of information about all DB sequences for all models in all apps."""
sequence_list = []
try:
assert os.environ.get(
"DJANGO_SETTINGS_MODULE"
), "No DJANGO_SETTINGS_MODULE variable defined"
except AssertionError as aerr:
raise aerr
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config,
self.connection.alias):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
for f in model._meta.local_fields:
if isinstance(f, models.AutoField):
sequence_list.append({
'table': model._meta.db_table,
'column': f.column
})
break # Only one AutoField is allowed per model, so don't bother continuing.
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.rel.through is None:
sequence_list.append({
'table': f.m2m_db_table(),
'column': None
})
return sequence_list
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
created_models = set()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False))
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables))
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=connection.features.can_rollback_ddl):
deferred_sql = []
for app_name, model_list in manifest.items():
for model in model_list:
if model._meta.proxy or not model._meta.managed:
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" % (app_name, model._meta.object_name)
)
with connection.schema_editor() as editor:
if self.verbosity >= 1:
self.stdout.write(" Creating table %s\n" % model._meta.db_table)
editor.create_model(model)
deferred_sql.extend(editor.deferred_sql)
editor.deferred_sql = []
created_models.add(model)
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL...\n")
for statement in deferred_sql:
cursor.execute(statement)
finally:
cursor.close()
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
try:
# Install custom SQL for the app (but only if this
# is a model we've just created)
if self.verbosity >= 1:
self.stdout.write(" Installing custom SQL...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(model, no_style(), connection)
if custom_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name)
)
try:
with transaction.atomic(using=connection.alias):
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install custom SQL for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e)
)
if self.show_traceback:
traceback.print_exc()
else:
if self.verbosity >= 3:
self.stdout.write(
" No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name)
)
finally:
cursor.close()
# Load initial_data fixtures (unless that has been disabled)
if self.load_initial_data:
for app_label in app_labels:
call_command(
'loaddata', 'initial_data', verbosity=self.verbosity,
database=connection.alias, app_label=app_label,
hide_empty=True,
)
return created_models
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
created_models = set()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False))
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.identifier_converter
# Note that if a model isn't managed we short-circuit and never try to install it
return not (
(converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)
)
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=connection.features.can_rollback_ddl):
with connection.schema_editor() as editor:
editor.execute("CREATE SCHEMA {}".format(settings.POSTGRES_TEMPLATE_SCHEMA))
statements = editor.connection.ops.prepare_sql_script(CLONE_SCHEMA)
for statement in statements:
editor.execute(statement, params=None)
schema_deferred_sql = {}
with connection.schema_editor() as editor:
schema_model = apps.get_model(settings.POSTGRES_SCHEMA_MODEL)
editor.create_model(schema_model, verbosity=self.verbosity)
schema_deferred_sql.update(editor.schema_deferred_sql)
editor.schema_deferred_sql = {}
created_models.add(schema_model)
for app_name, model_list in manifest.items():
for model in model_list:
if not model._meta.can_migrate(connection):
continue
if model in created_models:
continue # probably schema model
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" % (app_name, model._meta.object_name)
)
with connection.schema_editor() as editor:
editor.schema_deferred_sql.update(schema_deferred_sql)
editor.create_model(model, verbosity=self.verbosity)
schema_deferred_sql.update(editor.schema_deferred_sql)
editor.schema_deferred_sql = {}
created_models.add(model)
if self.verbosity >= 1:
self.stdout.write("\n Running deferred SQL...\n")
with connection.schema_editor() as editor:
editor.schema_deferred_sql = schema_deferred_sql
finally:
cursor.close()
return created_models
# -*- coding: utf-8 -*-
def sql_indexes(app, style, connection):
"Returns a list of the CREATE INDEX SQL statements for all models in the given app."
output = []
for model in router.get_migratable_models(app, connection.alias, include_auto_created=True):
output.extend(connection.creation.sql_indexes_for_model(model, style))
return output
def ordered_models_to_delete(app_config, connection):
"""Models of the given app to delete.
@return A tuple (models, loop_error).
'models' is a list of the models classes to delete ;
the order respects the dependencies between the models.
'loop_error' is a boolean which indicates dependencies loop error.
"""
from django.db import router
from django.utils.datastructures import OrderedSet
from creme.creme_core.utils.dependence_sort import dependence_sort, DependenciesLoopError
class ModelInfo:
# def __init__(self, model, dependencies, sql_cmd):
def __init__(self, model, dependencies):
self.model = model
self.dependencies = dependencies
# self.sql_cmd = sql_cmd
def __str__(self):
return 'ModelInfo(model={model}, dependencies={dependencies})'.format(
model=self.model.__name__,
dependencies=[d.__name__ for d in self.dependencies],
)
models_info = []
cursor = connection.cursor()
try:
table_names = set(connection.introspection.table_names(cursor))
app_models = OrderedSet(
router.get_migratable_models(
app_config,
connection.alias,
# include_auto_created=True,
# NB: the auto created tables are automatically
# deleted by schema_editor.delete_model(model)
include_auto_created=False,
))
for model in app_models:
meta = model._meta
if connection.introspection.table_name_converter(
meta.db_table) in table_names:
# dependencies = []
dependencies = set() # We use a set to avoid duplicates
for f in meta.local_fields:
# if f.rel:
if f.remote_field:
# related_model = f.rel.to
related_model = f.remote_field.model
# if related_model in app_models:
# NB: we avoid self-referencing (TODO: improve dependence_sort() ?)
if related_model is not model and related_model in app_models:
# dependencies.append(related_model)
dependencies.add(related_model)
models_info.append(
ModelInfo(
model=model,
dependencies=dependencies,
# sql_cmd=connection.creation.sql_destroy_model(model, [], style)[0],
))
finally:
cursor.close()
dep_error = False
try:
models_info = dependence_sort(
models_info,
get_key=lambda mi: mi.model,
get_dependencies=lambda mi: mi.dependencies,
)
except DependenciesLoopError:
dep_error = True
else:
models_info.reverse() # The dependencies must be deleted _after_
# return [mi.sql_cmd for mi in models_info], dep_error
return [mi.model for mi in models_info], dep_error
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
created_models = set()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config,
connection.alias,
include_auto_created=False))
for app_config in apps.get_app_configs()
if app_config.models_module is not None
and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
create_models = set(itertools.chain(*manifest.values()))
emit_pre_migrate_signal(create_models, self.verbosity,
self.interactive, connection.alias)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(
using=connection.alias,
savepoint=connection.features.can_rollback_ddl):
deferred_sql = []
for app_name, model_list in manifest.items():
for model in model_list:
if model._meta.proxy or not model._meta.managed:
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" %
(app_name, model._meta.object_name))
with connection.schema_editor() as editor:
if self.verbosity >= 1:
self.stdout.write(" Creating table %s\n" %
model._meta.db_table)
editor.create_model(model)
deferred_sql.extend(editor.deferred_sql)
editor.deferred_sql = []
created_models.add(model)
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL...\n")
for statement in deferred_sql:
cursor.execute(statement)
finally:
cursor.close()
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
try:
# Install custom SQL for the app (but only if this
# is a model we've just created)
if self.verbosity >= 1:
self.stdout.write(" Installing custom SQL...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(
model, no_style(), connection)
if custom_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing custom SQL for %s.%s model\n"
% (app_name, model._meta.object_name))
try:
with transaction.atomic(
using=connection.alias):
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install custom SQL for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e))
if self.show_traceback:
traceback.print_exc()
else:
if self.verbosity >= 3:
self.stdout.write(
" No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
finally:
cursor.close()
# Load initial_data fixtures (unless that has been disabled)
if self.load_initial_data:
for app_label in app_labels:
call_command(
'loaddata',
'initial_data',
verbosity=self.verbosity,
database=connection.alias,
app_label=app_label,
hide_empty=True,
)
return created_models
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config,
connection.alias,
include_auto_created=True))
for app_config in apps.get_app_configs()
if app_config.models_module is not None
and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
create_models = set(itertools.chain(*manifest.values()))
emit_pre_migrate_signal(create_models, self.verbosity,
self.interactive, connection.alias)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=False):
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" %
(app_name, model._meta.object_name))
sql, references = connection.creation.sql_create_model(
model, no_style(), seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto,
[]).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.
sql_for_pending_references(
refto, no_style(), pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, no_style(), pending_references))
if self.verbosity >= 1 and sql:
self.stdout.write(" Creating table %s\n" %
model._meta.db_table)
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
# We force a commit here, as that was the previous behavior.
# If you can prove we don't need this, remove it.
transaction.set_dirty(using=connection.alias)
finally:
cursor.close()
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
try:
# Install custom SQL for the app (but only if this
# is a model we've just created)
if self.verbosity >= 1:
self.stdout.write(" Installing custom SQL...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(
model, no_style(), connection)
if custom_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing custom SQL for %s.%s model\n"
% (app_name, model._meta.object_name))
try:
with transaction.commit_on_success_unless_managed(
using=connection.alias):
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install custom SQL for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e))
if self.show_traceback:
traceback.print_exc()
else:
if self.verbosity >= 3:
self.stdout.write(
" No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
if self.verbosity >= 1:
self.stdout.write(" Installing indexes...\n")
# Install SQL indices for all newly created models
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(
model, no_style())
if index_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing index for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
with transaction.commit_on_success_unless_managed(
using=connection.alias):
for sql in index_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install index for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e))
finally:
cursor.close()
# Load initial_data fixtures (unless that has been disabled)
if self.load_initial_data:
for app_label in app_labels:
call_command('loaddata',
'initial_data',
verbosity=self.verbosity,
database=connection.alias,
skip_validation=True,
app_label=app_label,
hide_empty=True)
return created_models
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=True))
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables))
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
create_models = set(itertools.chain(*manifest.values()))
emit_pre_migrate_signal(create_models, self.verbosity, self.interactive, connection.alias)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=connection.features.can_rollback_ddl):
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" % (app_name, model._meta.object_name)
)
sql, references = connection.creation.sql_create_model(model, no_style(), seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, no_style(), pending_references,
)
)
sql.extend(
connection.creation.sql_for_pending_references(
model, no_style(), pending_references
)
)
if self.verbosity >= 1 and sql:
self.stdout.write(" Creating table %s\n" % model._meta.db_table)
for statement in sql:
cursor.execute(statement)
tables.append(connection.introspection.table_name_converter(model._meta.db_table))
finally:
cursor.close()
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
try:
# Install custom SQL for the app (but only if this
# is a model we've just created)
if self.verbosity >= 1:
self.stdout.write(" Installing custom SQL...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(model, no_style(), connection)
if custom_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name)
)
try:
with transaction.atomic(using=connection.alias):
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install custom SQL for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e)
)
if self.show_traceback:
traceback.print_exc()
else:
if self.verbosity >= 3:
self.stdout.write(
" No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name)
)
if self.verbosity >= 1:
self.stdout.write(" Installing indexes...\n")
# Install SQL indices for all newly created models
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(model, no_style())
if index_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing index for %s.%s model\n" %
(app_name, model._meta.object_name)
)
savepoint = connection.features.can_rollback_ddl
try:
with transaction.atomic(using=connection.alias, savepoint=savepoint):
for sql in index_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install index for %s.%s model: %s\n" %
(app_name, model._meta.object_name, e)
)
finally:
cursor.close()
# Load initial_data fixtures (unless that has been disabled)
if self.load_initial_data:
for app_label in app_labels:
call_command(
'loaddata', 'initial_data', verbosity=self.verbosity,
database=connection.alias, skip_validation=True,
app_label=app_label, hide_empty=True,
)
return created_models
from collections import namedtuple
