def create_first_user(created_models, verbosity, db, app=None, **kwargs):
# this is super confusing
if app and app.__name__ != 'sentry.models':
return
if User not in created_models:
return
if hasattr(router, 'allow_migrate'):
if not router.allow_migrate(db, User):
return
else:
if not router.allow_syncdb(db, User):
return
if not kwargs.get('interactive', True):
return
import click
if not click.confirm('\nWould you like to create a user account now?', default=True):
# Not using `abort=1` because we don't want to exit out from further execution
click.echo('\nRun `sentry createuser` to do this later.\n')
return
from sentry.runner import call_command
call_command('sentry.runner.commands.createuser.createuser')
def _dump():
s_time = time.time()
from django.apps import apps
from django.db.models import get_model
excluded_models = set()
for exclude in EXCLUDED_MODELS:
app_label, model_name = exclude.split('.', 1)
model_obj = get_model(app_label, model_name)
excluded_models.add(model_obj)
app_list = [(c, None) for c in apps.get_app_configs() if c.label in
set(APPS_TO_HANDLE)]
objects = []
for model in serializers.sort_dependencies(app_list):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(DEFAULT_DB_ALIAS, model):
objects.extend(model._default_manager.using(DEFAULT_DB_ALIAS).all())
f_name = DUMP_NAME.format(datetime.now())
with closing(StringIO()) as compressed_data:
with gzip.GzipFile(filename=f_name, mode='wb',
fileobj=compressed_data) as compressor:
compressor.write(serializers.serialize('json', objects, indent=2,
use_natural_foreign_keys=True))
compressed_data.seek(0)
default_storage.save(DUMP_PATH + f_name + '.gz', compressed_data)
return '%d objects exported to %s in %d seconds'%(len(objects), f_name,
time.time() - s_time)
def db_router_allows_migrate(database, app_label, model_cls):
"""Return whether a database router allows migrate operations for a model.
This will only return ``True`` for Django 1.7 and newer and if the
router allows migrate operations. This is compatible with both the
Django 1.7 and 1.8+ versions of ``allow_migrate``.
Args:
database (unicode):
The name of the database.
app_label (unicode):
The application label.
model_cls (type):
The model class.
Returns:
bool:
``True`` if routers allow migrate for this model.
"""
if django.VERSION[:2] >= (1, 8):
return router.allow_migrate_model(database, model_cls)
elif django.VERSION[:2] == (1, 7):
return router.allow_migrate(database, model_cls)
else:
return False
def get_objects():
# Collate the objects to be serialized.
for model in sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
if related:
queryset = queryset.select_related()
serialized_objects = set()
for obj in queryset.iterator():
if related:
for rel_expression in related:
for rel_obj in get_related_objects(obj, rel_expression):
if rel_obj not in serialized_objects:
serialized_objects.add(rel_obj)
yield rel_obj
if obj not in serialized_objects:
yield obj
def get_objects():
for model in serializers.sort_dependencies(app_list):
if (not model._meta.proxy and model._meta.managed and
router.allow_migrate(self.connection.alias, model)):
queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
for obj in queryset.iterator():
yield obj
def create_default_user(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
try:
User = apps.get_model('dlabs_users', 'User')
except LookupError:
return
if not router.allow_migrate(using, User):
return
if not User.objects.using(using).exists():
if hasattr(settings, 'ADMINS'):
for admin in settings.ADMINS:
if verbosity >= 2:
print('Creating user %s' % admin[1])
name = admin[0].split(' ')
if len(name) >= 2:
User.objects.create_superuser(
email=admin[1],
first_name=name[0],
last_name=name[1],
password='******'
).save(using=using)
else:
User.objects.create_superuser(
email=admin[1],
password='******'
).save(using=using)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
old_apps = from_state.render()
new_apps = to_state.render()
old_model = old_apps.get_model(app_label, self.name)
new_model = new_apps.get_model(app_label, self.name)
if router.allow_migrate(schema_editor.connection.alias, new_model):
schema_editor.alter_db_table(new_model, old_model._meta.db_table, new_model._meta.db_table)
def load_fixture(name, using=DEFAULT_DB_ALIAS):
"""Progammatic way to load a fixture given some path. This does not
assume the path is a fixture within some app and assumes a full path.
"""
if os.path.isabs(name):
fixture_path = name
else:
fixture_path = full_fixture_path(name)
with open(fixture_path) as fixture:
objects = serializers.deserialize(FIXTURE_FORMAT, fixture, using=using)
try:
with transaction.atomic(using):
for obj in objects:
if (
hasattr(router, "allow_migrate") and
router.allow_migrate(using, obj.object.__class__)
) or (
hasattr(router, "allow_syncdb") and
router.allow_syncdb(using, obj.object.__class__)
):
obj.save(using=using)
except (DatabaseError, IntegrityError), e:
msg = u'Could not load {0}.{1}(pk={2}): {3}'.format(
obj.object._meta.app_label,
obj.object._meta.object_name, obj.object.pk, e)
raise e.__class__, e.__class__(msg), sys.exc_info()[2]
def database_forwards(self, app_label, schema_editor, from_state, to_state):
new_model = to_state.apps.get_model(app_label, self.name)
import pdb; pdb.set_trace()
sql = new_model._meta.sql
if router.allow_migrate(schema_editor.connection.alias, app_label):
self._run_sql(schema_editor, ("CREATE VIEW %s AS %s", (db_table, sql,)))
def django_table_names(self, only_existing=False):
"""
Returns a list of all table names that have associated Django models and
are in INSTALLED_APPS.
If only_existing is True, the resulting list will only include the tables
that actually exist in the database.
"""
from django.db import models, router
tables = set()
for app in models.get_apps():
for model in models.get_models(app):
if not model._meta.managed:
continue
if not router.allow_migrate(self.connection.alias, model):
continue
tables.add(model._meta.db_table)
tables.update(f.m2m_db_table() for f in model._meta.local_many_to_many)
tables = list(tables)
if only_existing:
existing_tables = self.table_names()
tables = [
t
for t in tables
if self.table_name_converter(t) in existing_tables
]
return tables
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
try:
Permission = apps.get_model('auth', 'Permission')
except LookupError:
return
if not router.allow_migrate(using, Permission):
return
# create a dummy content type for the app
from django.contrib.contenttypes.models import ContentType
appname = 'filebrowser_safe'
ct, created = ContentType.objects.get_or_create(model='', app_label=appname,
defaults={'name': appname})
# create each permission
permissions = (
('upload_file', 'Upload files'),
('overwrite_file', 'Overwrite files'),
('rename_file', 'Rename files'),
('delete_file', 'Delete files'),
)
for codename, name in permissions:
p, created = Permission.objects.get_or_create(codename=codename,
content_type__pk=ct.id,
defaults={'name': name, 'content_type': ct})
def sequence_list(self):
"Returns a list of information about all DB sequences for all models in all apps."
from django.db import models, router
apps = models.get_apps()
sequence_list = []
for app in apps:
for model in models.get_models(app):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
if not router.allow_migrate(self.connection.alias, model):
continue
for f in model._meta.local_fields:
if isinstance(f, models.AutoField):
sequence_list.append({'table': model._meta.db_table, 'column': f.column})
break # Only one AutoField is allowed per model, so don't bother continuing.
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.rel.through is None:
sequence_list.append({'table': f.m2m_db_table(), 'column': None})
return sequence_list
def create_permissions(app, created_models, verbosity, db=DEFAULT_DB_ALIAS, **kwargs):
try:
get_model('auth', 'Permission')
except UnavailableApp:
return
if not router.allow_migrate(db, auth_app.Permission):
return
from django.contrib.contenttypes.models import ContentType
app_models = get_models(app)
# This will hold the permissions we're looking for as
# (content_type, (codename, name))
searched_perms = list()
# The codenames and ctypes that should exist.
ctypes = set()
ctypes_for_models = ContentType.objects.get_for_models(
*app_models,
for_concrete_models=False
)
for klass, ctype in ctypes_for_models.iteritems():
# Force looking up the content types in the current database
# before creating foreign keys to them.
ctype = ContentType.objects.db_manager(db).get_for_model(klass)
ctypes.add(ctype)
for perm in _get_all_permissions(klass._meta, ctype):
searched_perms.append((ctype, perm))
# Find all the Permissions that have a content_type for a model we're
# looking for. We don't need to check for codenames since we already have
# a list of the ones we're going to create.
all_perms = set(auth_app.Permission.objects.using(db).filter(
content_type__in=ctypes,
).values_list(
"content_type", "codename"
))
perms = [
auth_app.Permission(codename=codename, name=name, content_type=ctype)
for ctype, (codename, name) in searched_perms
if (ctype.pk, codename) not in all_perms
]
# Validate the permissions before bulk_creation to avoid cryptic
# database error when the verbose_name is longer than 50 characters
permission_name_max_length = auth_app.Permission._meta.get_field('name').max_length
verbose_name_max_length = permission_name_max_length - 11 # len('Can change ') prefix
for perm in perms:
if len(perm.name) > permission_name_max_length:
raise exceptions.ValidationError(
"The verbose_name of %s is longer than %s characters" % (
perm.content_type,
verbose_name_max_length,
)
)
auth_app.Permission.objects.using(db).bulk_create(perms)
if verbosity >= 2:
for perm in perms:
print("Adding permission '%s'" % perm)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
if router.allow_migrate(schema_editor.connection.alias, app_label, **self.hints):
# We now execute the Python code in a context that contains a 'models'
# object, representing the versioned models as an app registry.
# We could try to override the global cache, but then people will still
# use direct imports, so we go with a documentation approach instead.
self.code(from_state.apps, schema_editor)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
old_apps = from_state.render()
new_apps = to_state.render()
old_model = old_apps.get_model(app_label, self.old_name)
new_model = new_apps.get_model(app_label, self.new_name)
if router.allow_migrate(schema_editor.connection.alias, new_model):
# Move the main table
schema_editor.alter_db_table(
new_model,
old_model._meta.db_table,
new_model._meta.db_table,
)
# Alter the fields pointing to us
related_objects = old_model._meta.get_all_related_objects()
related_m2m_objects = old_model._meta.get_all_related_many_to_many_objects()
for related_object in (related_objects + related_m2m_objects):
to_field = new_apps.get_model(
related_object.model._meta.app_label,
related_object.model._meta.object_name.lower(),
)._meta.get_field_by_name(related_object.field.name)[0]
schema_editor.alter_field(
related_object.model,
related_object.field,
to_field,
)
def create_default_site(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
try:
Site = apps.get_model('sites', 'Site')
except LookupError:
return
if not router.allow_migrate(using, Site):
return
if not Site.objects.exists():
# The default settings set SITE_ID = 1, and some tests in Django's test
# suite rely on this value. However, if database sequences are reused
# (e.g. in the test suite after flush/syncdb), it isn't guaranteed that
# the next id will be 1, so we coerce it. See #15573 and #16353. This
# can also crop up outside of tests - see #15346.
if verbosity >= 2:
print("Creating example.com Site object")
Site(pk=1, domain="example.com", name="example.com").save(using=using)
# We set an explicit pk instead of relying on auto-incrementation,
# so we need to reset the database sequence. See #17415.
sequence_sql = connections[using].ops.sequence_reset_sql(no_style(), [Site])
if sequence_sql:
if verbosity >= 2:
print("Resetting sequence")
with connections[using].cursor() as cursor:
for command in sequence_sql:
cursor.execute(command)
Site.objects.clear_cache()
def init_data(apps, verbosity, using, **kwargs):
Group = apps.get_model('community', 'Group')
Navigation = apps.get_model('community', 'Navigation')
CommunityUserProfileField = apps.get_model('community', 'CommunityUserProfileField')
if not router.allow_migrate(using, Group):
logger.info('Not allowed migration for Group.')
return
logger.info('allowing %s migrate? %s', using, repr(router.allow_migrate(using, Group)))
if Group.objects.count() > 0:
return
group, created = Group.objects.get_or_create(name='example',
longname='Example Group',
baseurl='www.example.com', )
group.save()
nav = Navigation(group=group,
label='Home',
href='/wiki/show/Start/',
urltype=0,
sortorder=10,
navigationType=0,
)
nav.save()
nav = Navigation(group=group,
label='Board',
href='/board/show/0/',
urltype=0,
sortorder=20,
navigationType=0,
)
nav.save()
CommunityUserProfileField(name='ICQ UIN',
regex='\d+',
sortorder=100, ).save()
CommunityUserProfileField(name='Jabber Id',
regex='.+@.+',
sortorder=200, ).save()
CommunityUserProfileField(name='Website URL',
regex='http://.*',
sortorder=300,
renderstring='<a href="%(value)s">%(value)s</a>', ).save()
def get_objects():
for model in sort_dependencies(app_list):
if not model._meta.proxy and router.allow_migrate(
self.connection.alias, model):
queryset = model._default_manager.using(
self.connection.alias).order_by(model._meta.pk.name)
for obj in queryset.iterator():
yield obj
def database_backwards(self, app_label, schema_editor, from_state,
to_state):
from_model = from_state.render().get_model(app_label, self.model_name)
to_model = to_state.render().get_model(app_label, self.model_name)
if router.allow_migrate(schema_editor.connection.alias, to_model):
schema_editor.add_field(
from_model,
to_model._meta.get_field_by_name(self.name)[0])
def create_permissions(app_config,
verbosity=2,
interactive=True,
using=DEFAULT_DB_ALIAS,
**kwargs):
if not app_config.models_module:
return
try:
Permission = apps.get_model('auth', 'Permission')
except LookupError:
return
if not router.allow_migrate(using, Permission):
return
from django.contrib.contenttypes.models import ContentType
# This will hold the permissions we're looking for as
# (content_type, (codename, name))
searched_perms = list()
# The codenames and ctypes that should exist.
ctypes = set()
for klass in app_config.get_models():
# Force looking up the content types in the current database
# before creating foreign keys to them.
ctype = ContentType.objects.db_manager(using).get_for_model(klass)
ctypes.add(ctype)
for perm in _get_all_permissions(klass._meta, ctype):
searched_perms.append((ctype, perm))
# Find all the Permissions that have a content_type for a model we're
# looking for. We don't need to check for codenames since we already have
# a list of the ones we're going to create.
all_perms = set(
Permission.objects.using(using).filter(
content_type__in=ctypes, ).values_list("content_type", "codename"))
perms = [
Permission(codename=codename, name=name, content_type=ctype)
for ctype, (codename, name) in searched_perms
if (ctype.pk, codename) not in all_perms
]
# Validate the permissions before bulk_creation to avoid cryptic
# database error when the verbose_name is longer than 50 characters
permission_name_max_length = Permission._meta.get_field('name').max_length
verbose_name_max_length = permission_name_max_length - 11 # len('Can change ') prefix
for perm in perms:
if len(perm.name) > permission_name_max_length:
raise exceptions.ValidationError(
"The verbose_name of %s is longer than %s characters" % (
perm.content_type,
verbose_name_max_length,
))
Permission.objects.using(using).bulk_create(perms)
if verbosity >= 2:
for perm in perms:
print("Adding permission '%s'" % perm)
def create_table(self, database, tablename):
cache = BaseDatabaseCache(tablename, {})
if not router.allow_migrate(database, cache.cache_model_class):
return
connection = connections[database]
if tablename in connection.introspection.table_names():
if self.verbosity > 0:
self.stdout.write("Cache table '%s' already exists." %
tablename)
return
fields = (
# "key" is a reserved word in MySQL, so use "cache_key" instead.
models.CharField(name='cache_key',
max_length=255,
unique=True,
primary_key=True),
models.TextField(name='value'),
models.DateTimeField(name='expires', db_index=True),
)
table_output = []
index_output = []
qn = connection.ops.quote_name
for f in fields:
field_output = [qn(f.name), f.db_type(connection=connection)]
field_output.append("%sNULL" % ("NOT " if not f.null else ""))
if f.primary_key:
field_output.append("PRIMARY KEY")
elif f.unique:
field_output.append("UNIQUE")
if f.db_index:
unique = "UNIQUE " if f.unique else ""
index_output.append("CREATE %sINDEX %s ON %s (%s);" %
(unique, qn('%s_%s' % (tablename, f.name)),
qn(tablename), qn(f.name)))
table_output.append(" ".join(field_output))
full_statement = ["CREATE TABLE %s (" % qn(tablename)]
for i, line in enumerate(table_output):
full_statement.append(
' %s%s' % (line, ',' if i < len(table_output) - 1 else ''))
full_statement.append(');')
with transaction.atomic(
using=database,
savepoint=connection.features.can_rollback_ddl):
with connection.cursor() as curs:
try:
curs.execute("\n".join(full_statement))
except DatabaseError as e:
raise CommandError(
"Cache table '%s' could not be created.\nThe error was: %s."
% (tablename, force_text(e)))
for statement in index_output:
curs.execute(statement)
if self.verbosity > 1:
self.stdout.write("Cache table '%s' created." % tablename)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
from_model = from_state.render().get_model(app_label, self.model_name)
to_model = to_state.render().get_model(app_label, self.model_name)
if router.allow_migrate(schema_editor.connection.alias, to_model):
schema_editor.alter_field(
from_model,
from_model._meta.get_field_by_name(self.name)[0],
to_model._meta.get_field_by_name(self.name)[0],
)
def load_app_configs(self, using):
migrated_apps = set()
for app_config in apps.get_app_configs():
app_label = app_config.label
if router.allow_migrate(using, app_label):
migrated_apps.add(app_config)
else:
self.stdout.write(f"app {app_label} not allow migration")
return migrated_apps
def allowed_to_migrate(self, connection_alias, model, hints=None):
"""
Returns if we're allowed to migrate the model.
"""
# Always skip if proxy, swapped out, or unmanaged.
if model and (model._meta.proxy or model._meta.swapped or not model._meta.managed):
return False
return router.allow_migrate(connection_alias, model, **(hints or {}))
def database_forwards(self, app_label, schema_editor, from_state,
to_state):
if router.allow_migrate(schema_editor.connection.alias, app_label,
**self.hints):
# We now execute the Python code in a context that contains a 'models'
# object, representing the versioned models as an app registry.
# We could try to override the global cache, but then people will still
# use direct imports, so we go with a documentation approach instead.
self.code(from_state.apps, schema_editor)
def database_backwards(self, app_label, schema_editor, from_state,
to_state):
if not router.allow_migrate(schema_editor.connection.alias, app_label):
return
schema_editor.execute("DROP EXTENSION %s" %
schema_editor.quote_name(self.name))
# Clear cached, stale oids.
get_hstore_oids.cache_clear()
get_citext_oids.cache_clear()
def fix_role_parenting_closure(app_config,
verbosity=2,
interactive=True,
using=DEFAULT_DB_ALIAS,
**kwargs):
'''Close the role parenting relation after migrations'''
if not router.allow_migrate(using, utils.get_role_parenting_model()):
return
utils.get_role_parenting_model().objects.update_transitive_closure()
def allowed_to_migrate(self, connection_alias, model, hints=None):
"""
Returns if we're allowed to migrate the model.
"""
# Always skip if proxy, swapped out, or unmanaged.
if model and (model._meta.proxy or model._meta.swapped
or not model._meta.managed):
return False
return router.allow_migrate(connection_alias, model, **(hints or {}))
def ensure_schema(self):
"""
Ensures the table exists and has the correct schema.
"""
if self.Migration._meta.db_table in self.connection.introspection.get_table_list(
self.connection.cursor()):
return
if router.allow_migrate(self.connection, self.Migration):
with self.connection.schema_editor() as editor:
editor.create_model(self.Migration)
def get_view_connection(cls, using):
"""
Returns connection for "using" database if migrations are allowed (via
router). Returns None if migrations are not allowed to indicate view
should not be used on the specified database.
Overwrite this method in subclass to customize, if needed.
"""
if router.allow_migrate(using, cls._meta.app_label):
return connections[using]
def allowed_to_migrate(self, connection_alias, model):
"""
Returns if we're allowed to migrate the model. Checks the router,
if it's a proxy, and if it's swapped out.
"""
return (
router.allow_migrate(connection_alias, model) and
not model._meta.proxy and
not model._meta.swapped
)
def emit_post_migrate(verbosity, interactive, database):
# Emit the post migrate signal. This allows individual applications to
# respond as if the database had been migrated from scratch.
all_models = []
for app in models.get_apps():
all_models.extend([
m for m in models.get_models(app, include_auto_created=True)
if router.allow_migrate(database, m)
])
emit_post_migrate_signal(set(all_models), verbosity, interactive, database)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
# RunPython has access to all models. Ensure that all models are
# reloaded in case any are delayed.
from_state.clear_delayed_apps_cache()
if router.allow_migrate(schema_editor.connection.alias, app_label, **self.hints):
# We now execute the Python code in a context that contains a 'models'
# object, representing the versioned models as an config registry.
# We could try to override the global cache, but then people will still
# use direct imports, so we go with a documentation approach instead.
self.code(from_state.apps, schema_editor)
def allowed_to_migrate(self, connection_alias, model):
"""
Returns if we're allowed to migrate the model. Checks the router,
if it's a proxy, if it's managed, and if it's swapped out.
"""
return (
not model._meta.proxy and
not model._meta.swapped and
model._meta.managed and
router.allow_migrate(connection_alias, model)
)
def get_all_related_objects(self, local_only=False, include_hidden=False,
include_proxy_eq=False):
using = router.db_for_write(self.model)
objs = models.base.Options.get_all_related_objects_with_model(
self,
local_only=local_only,
include_hidden=include_hidden,
include_proxy_eq=include_proxy_eq
)
objs = [k for k, v in objs if router.allow_migrate(using, k.model)]
return objs
def database_forwards(self, app_label, schema_editor, from_state, to_state):
old_apps = from_state.render()
new_apps = to_state.render()
old_model = old_apps.get_model(app_label, self.name)
new_model = new_apps.get_model(app_label, self.name)
if router.allow_migrate(schema_editor.connection.alias, new_model):
schema_editor.alter_index_together(
new_model,
getattr(old_model._meta, "index_together", set()),
getattr(new_model._meta, "index_together", set()),
)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
old_apps = from_state.render()
new_apps = to_state.render()
old_model = old_apps.get_model(app_label, self.name)
new_model = new_apps.get_model(app_label, self.name)
if router.allow_migrate(schema_editor.connection.alias, new_model):
schema_editor.alter_db_table(
new_model,
old_model._meta.db_table,
new_model._meta.db_table,
)
def emit_post_migrate(verbosity, interactive, database):
# Emit the post migrate signal. This allows individual applications to
# respond as if the database had been migrated from scratch.
all_models = []
for app in models.get_apps():
all_models.extend([
m for m in models.get_models(app, include_auto_created=True)
if router.allow_migrate(database, m)
])
emit_post_migrate_signal(set(all_models), verbosity, interactive,
database)
def installed_models(self, tables):
"Returns a set of all models represented by the provided list of table names."
from django.db import models, router
all_models = []
for app in models.get_apps():
for model in models.get_models(app):
if router.allow_migrate(self.connection.alias, model):
all_models.append(model)
tables = list(map(self.table_name_converter, tables))
return set([m for m in all_models if self.table_name_converter(m._meta.db_table) in tables])
def should_skip_detecting_model(migration, model):
"""
No need to detect tables for proxy models, unmanaged models, or
models that can't be migrated on the current database.
"""
return (model._meta.proxy or not model._meta.managed
or not router.allow_migrate(
self.connection.alias,
migration.app_label,
model_name=model._meta.model_name,
))
def database_forwards(self, app_label, schema_editor, from_state, to_state):
old_app_cache = from_state.render()
new_app_cache = to_state.render()
old_model = old_app_cache.get_model(app_label, self.name)
new_model = new_app_cache.get_model(app_label, self.name)
if router.allow_migrate(schema_editor.connection.alias, new_model):
schema_editor.alter_index_together(
new_model,
getattr(old_model._meta, "index_together", set()),
getattr(new_model._meta, "index_together", set()),
)
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
if not app_config.models_module:
return
try:
Permission = apps.get_model("auth", "Permission")
except LookupError:
return
if not router.allow_migrate(using, Permission):
return
from django.contrib.contenttypes.models import ContentType
# This will hold the permissions we're looking for as
# (content_type, (codename, name))
searched_perms = list()
# The codenames and ctypes that should exist.
ctypes = set()
for klass in app_config.get_models():
# Force looking up the content types in the current database
# before creating foreign keys to them.
ctype = ContentType.objects.db_manager(using).get_for_model(klass)
ctypes.add(ctype)
for perm in _get_all_permissions(klass._meta, ctype):
searched_perms.append((ctype, perm))
# Find all the Permissions that have a content_type for a model we're
# looking for. We don't need to check for codenames since we already have
# a list of the ones we're going to create.
all_perms = set(
Permission.objects.using(using).filter(content_type__in=ctypes).values_list("content_type", "codename")
)
perms = [
Permission(codename=codename, name=name, content_type=ct)
for ct, (codename, name) in searched_perms
if (ct.pk, codename) not in all_perms
]
# Validate the permissions before bulk_creation to avoid cryptic
# database error when the verbose_name is longer than 50 characters
permission_name_max_length = Permission._meta.get_field("name").max_length
verbose_name_max_length = permission_name_max_length - 11 # len('Can change ') prefix
for perm in perms:
if len(perm.name) > permission_name_max_length:
raise exceptions.ValidationError(
"The verbose_name of %s.%s is longer than %s characters"
% (perm.content_type.app_label, perm.content_type.model, verbose_name_max_length)
)
Permission.objects.using(using).bulk_create(perms)
if verbosity >= 2:
for perm in perms:
print("Adding permission '%s'" % perm)
def should_skip_detecting_model(migration, model):
"""
No need to detect tables for proxy models, unmanaged models, or
models that can't be migrated on the current database.
"""
return (
model._meta.proxy or not model._meta.managed or not
router.allow_migrate(
self.connection.alias, migration.app_label,
model_name=model._meta.model_name,
)
)
def load_label(self, fixture_label):
"""
Loads fixtures files for a given label.
"""
for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label):
_, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
open_method = self.compression_formats[cmp_fmt]
fixture = open_method(fixture_file, "r")
try:
self.fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
if self.verbosity >= 2:
self.stdout.write(
"Installing %s fixture '%s' from %s." % (ser_fmt, fixture_name, humanize(fixture_dir))
)
objects = serializers.deserialize(ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore)
for obj in objects:
objects_in_fixture += 1
if router.allow_migrate(self.using, obj.object.__class__):
loaded_objects_in_fixture += 1
self.models.add(obj.object.__class__)
try:
obj.save(using=self.using)
except (DatabaseError, IntegrityError) as e:
e.args = (
"Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s"
% {
"app_label": obj.object._meta.app_label,
"object_name": obj.object._meta.object_name,
"pk": obj.object.pk,
"error_msg": force_text(e),
},
)
raise
self.loaded_object_count += loaded_objects_in_fixture
self.fixture_object_count += objects_in_fixture
except Exception as e:
if not isinstance(e, CommandError):
e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),)
raise
finally:
fixture.close()
# Warn if the fixture we loaded contains 0 objects.
if objects_in_fixture == 0:
warnings.warn(
"No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, RuntimeWarning
)
def create_table(self, database, tablename):
cache = BaseDatabaseCache(tablename, {})
if not router.allow_migrate(database, cache.cache_model_class):
return
connection = connections[database]
if tablename in connection.introspection.table_names():
if self.verbosity > 0:
self.stdout.write("Cache table '%s' already exists." % tablename)
return
fields = (
# "key" is a reserved word in MySQL, so use "cache_key" instead.
models.CharField(name="cache_key", max_length=255, unique=True, primary_key=True),
models.TextField(name="value"),
models.DateTimeField(name="expires", db_index=True),
)
table_output = []
index_output = []
qn = connection.ops.quote_name
for f in fields:
field_output = [qn(f.name), f.db_type(connection=connection)]
field_output.append("%sNULL" % ("NOT " if not f.null else ""))
if f.primary_key:
field_output.append("PRIMARY KEY")
elif f.unique:
field_output.append("UNIQUE")
if f.db_index:
unique = "UNIQUE " if f.unique else ""
index_output.append(
"CREATE %sINDEX %s ON %s (%s);"
% (unique, qn("%s_%s" % (tablename, f.name)), qn(tablename), qn(f.name))
)
table_output.append(" ".join(field_output))
full_statement = ["CREATE TABLE %s (" % qn(tablename)]
for i, line in enumerate(table_output):
full_statement.append(" %s%s" % (line, "," if i < len(table_output) - 1 else ""))
full_statement.append(");")
with transaction.atomic(using=database, savepoint=connection.features.can_rollback_ddl):
with connection.cursor() as curs:
try:
curs.execute("\n".join(full_statement))
except DatabaseError as e:
raise CommandError(
"Cache table '%s' could not be created.\nThe error was: %s." % (tablename, force_text(e))
)
for statement in index_output:
curs.execute(statement)
if self.verbosity > 1:
self.stdout.write("Cache table '%s' created." % tablename)
def installed_models(self, tables):
"Returns a set of all models represented by the provided list of table names."
from django.db import models, router
all_models = []
for app in models.get_apps():
for model in models.get_models(app):
if router.allow_migrate(self.connection.alias, model):
all_models.append(model)
tables = list(map(self.table_name_converter, tables))
return set([
m for m in all_models
if self.table_name_converter(m._meta.db_table) in tables
])
def database_forwards(self, app_label, schema_editor, from_state, to_state):
from_model = from_state.render().get_model(app_label, self.model_name)
to_model = to_state.render().get_model(app_label, self.model_name)
if router.allow_migrate(schema_editor.connection.alias, to_model):
field = to_model._meta.get_field_by_name(self.name)[0]
if not self.preserve_default:
field.default = self.field.default
schema_editor.add_field(
from_model,
field,
)
if not self.preserve_default:
field.default = NOT_PROVIDED
def load_label(self, fixture_label):
"""
Loads fixtures files for a given label.
"""
for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label):
_, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
open_method, mode = self.compression_formats[cmp_fmt]
fixture = open_method(fixture_file, mode)
try:
self.fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
if self.verbosity >= 2:
self.stdout.write("Installing %s fixture '%s' from %s." %
(ser_fmt, fixture_name, humanize(fixture_dir)))
objects = serializers.deserialize(ser_fmt, fixture,
using=self.using, ignorenonexistent=self.ignore)
for obj in objects:
objects_in_fixture += 1
if router.allow_migrate(self.using, obj.object.__class__):
loaded_objects_in_fixture += 1
self.models.add(obj.object.__class__)
try:
obj.save(using=self.using)
except (DatabaseError, IntegrityError) as e:
e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
'app_label': obj.object._meta.app_label,
'object_name': obj.object._meta.object_name,
'pk': obj.object.pk,
'error_msg': force_text(e)
},)
raise
self.loaded_object_count += loaded_objects_in_fixture
self.fixture_object_count += objects_in_fixture
except Exception as e:
if not isinstance(e, CommandError):
e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),)
raise
finally:
fixture.close()
# Warn if the fixture we loaded contains 0 objects.
if objects_in_fixture == 0:
warnings.warn(
"No fixture data found for '%s'. (File format may be "
"invalid.)" % fixture_name,
RuntimeWarning
)
def database_forwards(self, app_label, schema_editor, from_state,
to_state):
if schema_editor.connection.vendor != "postgresql" or not router.allow_migrate(
schema_editor.connection.alias, app_label):
return
schema_editor.execute("CREATE EXTENSION IF NOT EXISTS %s" %
schema_editor.quote_name(self.name))
# Clear cached, stale oids.
get_hstore_oids.cache_clear()
get_citext_oids.cache_clear()
# Registering new type handlers cannot be done before the extension is
# installed, otherwise a subsequent data migration would use the same
# connection.
register_type_handlers(schema_editor.connection)
def database_forwards(self, app_label, schema_editor, from_state,
to_state) -> None:
if router.allow_migrate(schema_editor.connection.alias, app_label,
**self.hints):
base_sql = _get_reset_sql()
with schema_editor.connection.cursor() as cursor:
cursor.execute(base_sql)
rows = cursor.fetchall()
sql = "\n".join(x[0] for x in rows)
self._run_sql(schema_editor, sql)
def get_objects_into_chunks(max_records_per_chunk):
# Collate the objects to be serialized.
model_count = 0
for model in sort_dependencies(app_list.items()):
model_count += 1
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(
using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(
model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
items_total = queryset.count()
chunk_count = 0
chunks_total = (items_total / max_records_per_chunk) + (
1 if (items_total % max_records_per_chunk) > 0 else 0)
for chunk_num in range(0, chunks_total):
output_objects = queryset[chunk_num *
max_records_per_chunk:
(chunk_num + 1) *
max_records_per_chunk]
chunk_count += 1
dump_file_name = output_folder + "/%04d_%04d.json" % (
model_count, chunk_count)
print "Dumping file: %s [%d/%d] for %s" % (
dump_file_name, chunk_count, chunks_total,
model.__name__)
output = serializers.serialize(
format,
output_objects,
indent=indent,
use_natural_keys=use_natural_keys)
try:
with open(dump_file_name, "w") as dumpfile:
dumpfile.write(output)
except Exception as e:
if show_traceback:
raise
raise CommandError("Unable to write file: %s" % e)
def post_migrate_update_rbac(app_config, verbosity=2, interactive=True,
using=DEFAULT_DB_ALIAS, **kwargs):
# be sure new objects names are localized using the default locale
from .management import update_ou_admin_roles, update_ous_admin_roles, \
update_content_types_roles
if not router.allow_migrate(using, get_role_model()):
return
with override(settings.LANGUAGE_CODE):
with transaction.atomic():
with defer_update_transitive_closure():
update_content_types_roles()
update_ous_admin_roles()
def sqlcomment_post_migrate(app_config,
using=DEFAULT_DB_ALIAS,
apps=apps,
**kwargs):
app_label = app_config.label
if not router.allow_migrate(using, app_label):
return
stmts = list(sqlcomment_statements(app_config, using=using))
if stmts:
with connections[using].cursor() as cursor:
with transaction.atomic():
for stmt in stmts:
cursor.execute(stmt)
def create_default_permissions(app_config,
verbosity=2,
interactive=True,
using=DEFAULT_DB_ALIAS,
**kwargs):
from .models import CHANGE_PASSWORD_OP, RESET_PASSWORD_OP, ACTIVATE_OP, CHANGE_EMAIL_OP
if not router.allow_migrate(using, get_ou_model()):
return
with override(settings.LANGUAGE_CODE):
get_operation(CHANGE_PASSWORD_OP)
get_operation(RESET_PASSWORD_OP)
get_operation(ACTIVATE_OP)
get_operation(CHANGE_EMAIL_OP)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
from_model = from_state.render().get_model(app_label, self.model_name)
to_model = to_state.render().get_model(app_label, self.model_name)
if router.allow_migrate(schema_editor.connection.alias, to_model):
from_field = from_model._meta.get_field_by_name(self.name)[0]
to_field = to_model._meta.get_field_by_name(self.name)[0]
# If the field is a relatedfield with an unresolved rel.to, just
# set it equal to the other field side. Bandaid fix for AlterField
# migrations that are part of a RenameModel change.
if from_field.rel and from_field.rel.to:
if isinstance(from_field.rel.to, six.string_types):
from_field.rel.to = to_field.rel.to
elif isinstance(to_field.rel.to, six.string_types):
to_field.rel.to = from_field.rel.to
schema_editor.alter_field(from_model, from_field, to_field)
def create_base_operations(app_config,
verbosity=2,
interactive=True,
using=DEFAULT_DB_ALIAS,
**kwargs):
'''Create some basic operations, matching permissions from Django'''
if not router.allow_migrate(using, models.Operation):
return
utils.get_operation(models.ADD_OP)
utils.get_operation(models.CHANGE_OP)
utils.get_operation(models.DELETE_OP)
utils.get_operation(models.VIEW_OP)
utils.get_operation(models.ADMIN_OP)
utils.get_operation(models.SEARCH_OP)
def get_objects():
# Collate the objects to be serialized.
for model in sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
for obj in queryset.iterator():
yield obj
def post_migrate_update_rbac(app_config,
verbosity=2,
interactive=True,
using=DEFAULT_DB_ALIAS,
**kwargs):
# be sure new objects names are localized using the default locale
from .management import update_ou_admin_roles, update_ous_admin_roles, \
update_content_types_roles
if not router.allow_migrate(using, get_role_model()):
return
with override(settings.LANGUAGE_CODE):
with transaction.atomic():
with defer_update_transitive_closure():
update_content_types_roles()
update_ous_admin_roles()
def handle_label(self, tablename, **options):
db = options.get('database')
cache = BaseDatabaseCache(tablename, {})
if not router.allow_migrate(db, cache.cache_model_class):
return
connection = connections[db]
fields = (
# "key" is a reserved word in MySQL, so use "cache_key" instead.
models.CharField(name='cache_key',
max_length=255,
unique=True,
primary_key=True),
models.TextField(name='value'),
models.DateTimeField(name='expires', db_index=True),
)
table_output = []
index_output = []
qn = connection.ops.quote_name
for f in fields:
field_output = [qn(f.name), f.db_type(connection=connection)]
field_output.append("%sNULL" % ("NOT " if not f.null else ""))
if f.primary_key:
field_output.append("PRIMARY KEY")
elif f.unique:
field_output.append("UNIQUE")
if f.db_index:
unique = "UNIQUE " if f.unique else ""
index_output.append("CREATE %sINDEX %s ON %s (%s);" % \
(unique, qn('%s_%s' % (tablename, f.name)), qn(tablename),
qn(f.name)))
table_output.append(" ".join(field_output))
full_statement = ["CREATE TABLE %s (" % qn(tablename)]
for i, line in enumerate(table_output):
full_statement.append(
' %s%s' % (line, ',' if i < len(table_output) - 1 else ''))
full_statement.append(');')
with transaction.commit_on_success_unless_managed():
curs = connection.cursor()
try:
curs.execute("\n".join(full_statement))
except DatabaseError as e:
raise CommandError(
"Cache table '%s' could not be created.\nThe error was: %s."
% (tablename, force_text(e)))
for statement in index_output:
curs.execute(statement)
