def ready(self):
mapping_module = getattr(settings, 'SEEKER_MAPPING_MODULE', 'mappings')
mappings = getattr(settings, 'SEEKER_MAPPINGS', [])
module_only = getattr(settings, 'SEEKER_MODULE_ONLY', True)
if mappings:
# Keep a mapping of app module to app label (project.app.subapp -> subapp)
app_lkup = {app.name: app.label for app in apps.get_app_configs()}
for mapping in mappings:
mapping_cls = import_class(mapping)
# Figure out which app_label to use based on the longest matching module prefix.
app_label = None
for prefix in sorted(app_lkup):
if mapping.startswith(prefix):
app_label = app_lkup[prefix]
register(mapping_cls, app_label=app_label)
else:
if not mapping_module:
return
for app in apps.get_app_configs():
try:
module = '%s.%s' % (app.name, mapping_module)
imported_module = importlib.import_module(module)
clsmembers = inspect.getmembers(imported_module, lambda member: inspect.isclass(member) and issubclass(member, Indexable))
for name, cls in clsmembers:
if module_only and cls.__module__ != module:
logger.debug('Skipping registration of %s.%s (defined outside %s)', cls.__module__, name, module)
continue
register(cls, app_label=app.label)
except ImportError:
pass
def find_templates():
template_classes.clear()
layout_classes.clear()
for app in apps.get_app_configs():
if os.path.isfile(os.path.join(app.path, 'mails.py')):
import_module('{}.mails'.format(app.name))
for cls in all_template_classes:
if cls.abstract:
continue
found = False
for app in apps.get_app_configs():
if in_module(cls, app.module):
found = True
break
if found:
template_classes.append(cls)
for cls in all_layout_classes:
if cls.abstract:
continue
found = False
for app in apps.get_app_configs():
if in_module(cls, app.module):
found = True
break
if found:
layout_classes.append(cls)
def cms_editor(apps, schema_editor):
from django.contrib.contenttypes.management import update_contenttypes
from django.apps import apps as configured_apps
for app in configured_apps.get_app_configs():
update_contenttypes(app, interactive=True, verbosity=0)
from django.contrib.auth.management import create_permissions
for app in configured_apps.get_app_configs():
create_permissions(app, verbosity=0)
create_perms(apps)
def autodiscover_tasks(self):
try:
from django.apps import apps
except ImportError:
return self._settings.INSTALLED_APPS
else:
return [config.name for config in apps.get_app_configs()]
def _get_provide_specs_from_apps(category):
"""
Load provide spec strings from installed `shoop.apps.AppConfig`s.
This function takes care to order the spec list to match the order
the apps were enabled in `INSTALLED_APPS` to avoid nondeterministic
failures that caused by different values of `PYTHONHASHSEED`
(and the resulting change of dict iteration order).
:param category: Provide category name
:type category: str
:return: List of spec strings.
:rtype: list[str]
"""
if category not in _provide_specs: # (Re)load required?
provide_list = []
for app_config in apps.get_app_configs():
if not isinstance(app_config, AppConfig):
# No need to look at non-Shoop AppConfigen.
continue
spec_list = app_config.provides.get(category, ())
for spec in spec_list: # Insert in order without duplicates...
if spec not in provide_list:
provide_list.append(spec)
_provide_specs[category] = provide_list
return _provide_specs[category]
def model_generator():
"""
Yields all models of all apps.
"""
for app_config in apps.get_app_configs():
for model in app_config.get_models():
yield model
def get_installed_libraries():
"""
Return the built-in template tag libraries and those from installed
applications. Libraries are stored in a dictionary where keys are the
individual module names, not the full module paths. Example:
django.templatetags.i18n is stored as i18n.
"""
libraries = {}
candidates = ['django.templatetags']
candidates.extend(
'%s.templatetags' % app_config.name
for app_config in apps.get_app_configs())
for candidate in candidates:
try:
pkg = import_module(candidate)
except ImportError:
# No templatetags package defined. This is safe to ignore.
continue
if hasattr(pkg, '__path__'):
for name in get_package_libraries(pkg):
libraries[name[len(candidate) + 1:]] = name
return libraries
def get_installed_app_labels_with_migrations():
""" Get the app labels, because settings.INSTALLED_APPS doesn't necessarily give us the labels.
Remove django.contrib.contenttypes because we want it to run before us.
Return list of tuples like ('admin', '__first__')
"""
from django.apps import apps
apps_with_migrations = []
for app in apps.get_app_configs():
if app.label == 'contenttypes': continue # Ignore the contenttypes app
migrations_module = MigrationLoader.migrations_module(app.label)
try:
module = import_module(migrations_module)
except ImportError:
continue
if not hasattr(module, "__path__"):
continue
# Make sure there are python files in the migration folder
has_files = bool(x for x in os.listdir(module.__path__[0]) if x.endswith(".py"))
if not has_files:
continue
apps_with_migrations.append(app.label)
return [(x, '__first__') for x in apps_with_migrations]
def autodiscover_modules(*args, **kwargs):
"""
Auto-discover INSTALLED_APPS modules and fail silently when
not present. This forces an import on them to register any admin bits they
may want.
You may provide a register_to keyword parameter as a way to access a
registry. This register_to object must have a _registry instance variable
to access it.
"""
from django.apps import apps
register_to = kwargs.get('register_to')
for app_config in apps.get_app_configs():
for module_to_search in args:
# Attempt to import the app's module.
try:
if register_to:
before_import_registry = copy.copy(register_to._registry)
import_module('%s.%s' % (app_config.name, module_to_search))
except:
# Reset the registry to the state before the last import
# as this import will have to reoccur on the next request and
# this could raise NotRegistered and AlreadyRegistered
# exceptions (see #8245).
if register_to:
register_to._registry = before_import_registry
# Decide whether to bubble up this error. If the app just
# doesn't have the module in question, we can ignore the error
# attempting to import it, otherwise we want it to bubble up.
if module_has_submodule(app_config.module, module_to_search):
raise
def fixture_dirs(self):
"""
Return a list of fixture directories.
The list contains the 'fixtures' subdirectory of each installed
application, if it exists, the directories in FIXTURE_DIRS, and the
current directory.
"""
dirs = []
fixture_dirs = settings.FIXTURE_DIRS
if len(fixture_dirs) != len(set(fixture_dirs)):
raise ImproperlyConfigured("settings.FIXTURE_DIRS contains duplicates.")
for app_config in apps.get_app_configs():
app_label = app_config.label
app_dir = os.path.join(app_config.path, 'fixtures')
if app_dir in fixture_dirs:
raise ImproperlyConfigured(
"'%s' is a default fixture directory for the '%s' app "
"and cannot be listed in settings.FIXTURE_DIRS." % (app_dir, app_label)
)
if self.app_label and app_label != self.app_label:
continue
if os.path.isdir(app_dir):
dirs.append(app_dir)
dirs.extend(list(fixture_dirs))
dirs.append('')
dirs = [os.path.abspath(os.path.realpath(d)) for d in dirs]
return dirs
def gen_app_versions(self):
for app_config in apps.get_app_configs():
name = app_config.verbose_name
app = app_config.module
version = self.get_app_version(app)
if version:
yield app.__name__, name, version
def get_commands():
"""
Returns a dictionary mapping command names to their callback applications.
This works by looking for a management.commands package in django.core, and
in each installed application -- if a commands package exists, all commands
in that package are registered.
Core commands are always included. If a settings module has been
specified, user-defined commands will also be included.
The dictionary is in the format {command_name: app_name}. Key-value
pairs from this dictionary can then be used in calls to
load_command_class(app_name, command_name)
If a specific version of a command must be loaded (e.g., with the
startapp command), the instantiated module can be placed in the
dictionary in place of the application name.
The dictionary is cached on the first call and reused on subsequent
calls.
"""
commands = {name: 'django.core' for name in find_commands(upath(__path__[0]))}
if not settings.configured:
return commands
for app_config in reversed(list(apps.get_app_configs())):
path = os.path.join(app_config.path, 'management')
commands.update({name: app_config.name for name in find_commands(path)})
return commands
def serialize_db_to_string(self):
"""
Serializes all data in the database into a JSON string.
Designed only for test runner usage; will not handle large
amounts of data.
"""
# Build list of all apps to serialize
from django.db.migrations.loader import MigrationLoader
loader = MigrationLoader(self.connection)
app_list = []
for app_config in apps.get_app_configs():
if (
app_config.models_module is not None and
app_config.label in loader.migrated_apps and
app_config.name not in settings.TEST_NON_SERIALIZED_APPS
):
app_list.append((app_config, None))
# Make a function to iteratively return every object
def get_objects():
for model in serializers.sort_dependencies(app_list):
if (not model._meta.proxy and model._meta.managed and
router.allow_migrate(self.connection.alias, model)):
queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
for obj in queryset.iterator():
yield obj
# Serialize to a string
out = StringIO()
serializers.serialize("json", get_objects(), indent=None, stream=out)
return out.getvalue()
def emit_post_migrate(verbosity, interactive, database):
# Emit the post migrate signal. This allows individual applications to
# respond as if the database had been migrated from scratch.
all_models = []
for app_config in apps.get_app_configs():
all_models.extend(router.get_migratable_models(app_config, database, include_auto_created=True))
emit_post_migrate_signal(set(all_models), verbosity, interactive, database)
def autodiscover():
"""
Iterate over django.apps.get_app_configs() and discover
versatileimagefield.py modules.
"""
from importlib import import_module
from django.apps import apps
from django.utils.module_loading import module_has_submodule
for app_config in apps.get_app_configs():
# Attempt to import the app's module.
try:
before_import_sizedimage_registry = copy.copy(
versatileimagefield_registry._sizedimage_registry
)
before_import_filter_registry = copy.copy(
versatileimagefield_registry._filter_registry
)
import_module('%s.versatileimagefield' % app_config.name)
except:
# Reset the versatileimagefield_registry to the state before the
# last import as this import will have to reoccur on the next
# request and this could raise NotRegistered and AlreadyRegistered
# exceptions (see django ticket #8245).
versatileimagefield_registry._sizedimage_registry = \
before_import_sizedimage_registry
versatileimagefield_registry._filter_registry = \
before_import_filter_registry
# Decide whether to bubble up this error. If the app just
# doesn't have the module in question, we can ignore the error
# attempting to import it, otherwise we want it to bubble up.
if module_has_submodule(app_config.module, 'versatileimagefield'):
raise
def django_table_names(self, only_existing=False, include_views=True):
"""
Returns a list of all table names that have associated Django models and
are in INSTALLED_APPS.
If only_existing is True, the resulting list will only include the tables
that actually exist in the database.
"""
from django.apps import apps
from django.db import router
tables = set()
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config, self.connection.alias):
if not model._meta.managed:
continue
tables.add(model._meta.db_table)
tables.update(f.m2m_db_table() for f in model._meta.local_many_to_many)
tables = list(tables)
if only_existing:
existing_tables = self.table_names(include_views=include_views)
tables = [
t
for t in tables
if self.table_name_converter(t) in existing_tables
]
return tables
def test_dependency_edge_case(self):
""" Test an edge case where migration dependency is fully migrated """
# Fully migrate the app the collected migration depends on
call_command('migrate', 'blog', verbosity=0)
module = 'django_migrate_project.management.commands.applymigrations'
get_app_configs_path = module + '.apps.get_app_configs'
app_configs = apps.get_app_configs()
with mock.patch(get_app_configs_path) as get_app_configs:
# NOTE: This is reversed to get full line statement coverage,
# where it needs 'cookbook' to be processed before 'blog'
get_app_configs.return_value = list(reversed(list(app_configs)))
connection = connections[DEFAULT_DB_ALIAS]
loader = MigrationLoader(connection)
applied_migrations = copy(loader.applied_migrations)
call_command('applymigrations', verbosity=0,
input_dir=DEPENDENCY_EDGE_MIGRATION_DIR)
# Check that database changed
loader = MigrationLoader(connection)
self.assertNotEqual(loader.applied_migrations, applied_migrations)
def get_env():
"""Configure and return a jinja2 Environment."""
global _env
if _env:
return _env
# Mimic Django's setup by loading templates from directories in
# TEMPLATE_DIRS and packages in INSTALLED_APPS.
loaders = [jinja2.FileSystemLoader(d) for d in settings.TEMPLATE_DIRS]
loaders += [jinja2.PackageLoader(c.name) for c in apps.get_app_configs()]
opts = {
'trim_blocks': True,
'extensions': ['jinja2.ext.i18n'],
'autoescape': True,
'auto_reload': settings.DEBUG,
'loader': jinja2.ChoiceLoader(loaders),
}
if hasattr(settings, 'JINJA_CONFIG'):
if hasattr(settings.JINJA_CONFIG, '__call__'):
config = settings.JINJA_CONFIG()
else:
config = settings.JINJA_CONFIG
opts.update(config)
e = Environment(**opts)
# Install null translations since gettext isn't always loaded up during
# testing.
if ('jinja2.ext.i18n' in e.extensions or
'jinja2.ext.InternationalizationExtension' in e.extensions):
e.install_null_translations()
_env = e
return e
def sequence_list(self):
"Returns a list of information about all DB sequences for all models in all apps."
from django.apps import apps
from django.db import models, router
sequence_list = []
for app_config in apps.get_app_configs():
for model in router.get_migratable_models(app_config, self.connection.alias):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
for f in model._meta.local_fields:
if isinstance(f, models.AutoField):
sequence_list.append({'table': model._meta.db_table, 'column': f.column})
break # Only one AutoField is allowed per model, so don't bother continuing.
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.remote_field.through is None:
sequence_list.append({'table': f.m2m_db_table(), 'column': None})
return sequence_list
def handle(self, *args, **options):
testable_apps = []
for app in apps.get_app_configs():
if not app.name.startswith("django."):
testable_apps.append(app.name)
call_command('test', *testable_apps, **options)
return
def runjobs_by_signals(self, when, options):
""" Run jobs from the signals """
# Thanks for Ian Holsman for the idea and code
from django_extensions.management import signals
from django.conf import settings
verbosity = options["verbosity"]
for app_name in settings.INSTALLED_APPS:
try:
__import__(app_name + '.management', '', '', [''])
except ImportError:
pass
for app in (app.models_module for app in apps.get_app_configs() if app.models_module):
if verbosity > 1:
app_name = '.'.join(app.__name__.rsplit('.')[:-1])
print("Sending %s job signal for: %s" % (when, app_name))
if when == 'minutely':
signals.run_minutely_jobs.send(sender=app, app=app)
elif when == 'quarter_hourly':
signals.run_quarter_hourly_jobs.send(sender=app, app=app)
elif when == 'hourly':
signals.run_hourly_jobs.send(sender=app, app=app)
elif when == 'daily':
signals.run_daily_jobs.send(sender=app, app=app)
elif when == 'weekly':
signals.run_weekly_jobs.send(sender=app, app=app)
elif when == 'monthly':
signals.run_monthly_jobs.send(sender=app, app=app)
elif when == 'yearly':
signals.run_yearly_jobs.send(sender=app, app=app)
def get_apps_from_cache():
try:
from django.apps import apps
return [app.models_module for app in apps.get_app_configs() if app.models_module]
except ImportError:
from django.db.models.loading import cache
return cache.get_apps()
def ready(self):
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
installed_apps = [app_config.name for app_config in apps.get_app_configs()]
app.autodiscover_tasks(lambda: installed_apps, force=True)
if hasattr(settings, 'RAVEN_CONFIG'):
# Celery signal registration
from raven import Client as RavenClient
from raven.contrib.celery import register_signal as raven_register_signal
from raven.contrib.celery import register_logger_signal as raven_register_logger_signal
raven_client = RavenClient(dsn=settings.RAVEN_CONFIG['DSN'])
raven_register_logger_signal(raven_client)
raven_register_signal(raven_client)
if hasattr(settings, 'OPBEAT'):
from opbeat.contrib.django.models import client as opbeat_client
from opbeat.contrib.django.models import logger as opbeat_logger
from opbeat.contrib.django.models import register_handlers as opbeat_register_handlers
from opbeat.contrib.celery import register_signal as opbeat_register_signal
try:
opbeat_register_signal(opbeat_client)
except Exception as e:
opbeat_logger.exception('Failed installing celery hook: %s' % e)
if 'opbeat.contrib.django' in settings.INSTALLED_APPS:
opbeat_register_handlers()
def get_test_labels(cls, tests):
"""Get a list of app labels and possibly tests for the db app optimizer
This should be called after `loadTestsFromNames` has been called.
"""
test_apps = set(app.name for app in apps.get_app_configs()
if app.name not in settings.APPS_TO_EXCLUDE_FROM_TESTS
and not app.name.startswith('django.'))
if not cls.user_specified_test_names:
return [AppConfig.create(app).label for app in test_apps]
def iter_names(test):
# a.b.c -> a.b.c, a.b, a
if isinstance(test.context, type):
name = test.context.__module__
elif isinstance(test.context, types.ModuleType):
name = test.context.__name__
else:
raise RuntimeError("unknown test type: {!r}".format(test))
parts = name.split(".")
num_parts = len(parts)
for i in range(num_parts):
yield ".".join(parts[:num_parts - i])
labels = set()
for test in tests:
for name in iter_names(test):
if name in test_apps:
labels.add((AppConfig.create(name).label, test.context))
break
return list(labels)
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
created_models = set()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config, connection.alias, include_auto_created=False))
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not (
(converter(opts.db_table) in tables) or
(opts.auto_created and converter(opts.auto_created._meta.db_table) in tables)
)
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models
)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=connection.features.can_rollback_ddl):
deferred_sql = []
for app_name, model_list in manifest.items():
for model in model_list:
if not model._meta.can_migrate(connection):
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" % (app_name, model._meta.object_name)
)
with connection.schema_editor() as editor:
if self.verbosity >= 1:
self.stdout.write(" Creating table %s\n" % model._meta.db_table)
editor.create_model(model)
deferred_sql.extend(editor.deferred_sql)
editor.deferred_sql = []
created_models.add(model)
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL...\n")
for statement in deferred_sql:
cursor.execute(statement)
finally:
cursor.close()
return created_models
def load_menus(c):
"""
load_menus loops through INSTALLED_APPS and loads the menu.py
files from them.
"""
# we don't need to do this more than once
if c.loaded:
return
# Fetch all installed app names
app_names = settings.INSTALLED_APPS
if apps:
app_names = [app_config.name for app_config in apps.get_app_configs()]
# loop through our INSTALLED_APPS
for app in app_names:
# skip any django apps
if app.startswith("django."):
continue
menu_module = "%s.menus" % app
try:
__import__(menu_module, fromlist=["menu"])
except ImportError:
pass
c.loaded = True
def handle(self, addrport, **kwargs):
# In auto-reload mode, migrations will be re-made and the server will be reloaded (fixtures are not reloaded)
for app in apps.get_app_configs():
if app.name in settings.SCAPL_INSTALLED_APPS:
call_command('makemigrations', app.label, interactive=False)
call_command('migrate')
try:
executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
executor.migration_plan(executor.loader.graph.leaf_nodes())
except Exception as e:
logger.error("Something failed with migrations execution")
for l in str(e).split():
logger.error(l)
return
if os.environ.get('RUN_MAIN') != 'true' and kwargs.get('fixtures'):
dir = settings.SMUGGLER_FIXTURE_DIR
for fn in sorted([f for f in os.listdir(dir) if f.endswith('.json')]):
fixture = os.path.join(dir, fn)
print("{}, ".format(fixture)),
try:
call_command('loaddata', fixture, ignorenonexistent=True)
except Exception as e:
print("No fixture loaded")
if isinstance(e, TypeError):
logger.warning(" +---> Please refer to 'BUGFIX' in the 'data' folder for this kind of error")
else:
with open(fixture) as f:
if f.read().strip(" \n[]") == "":
logger.warning(" +---> This fixture is emtpy")
call_command('runserver', addrport)
def get_migration_status(**options):
# type: (**Any) -> str
verbosity = options.get('verbosity', 1)
for app_config in apps.get_app_configs():
if module_has_submodule(app_config.module, "management"):
import_module('.management', app_config.name)
app_labels = [options['app_label']] if options.get('app_label') else None
db = options.get('database', DEFAULT_DB_ALIAS)
out = StringIO()
call_command(
'showmigrations',
'--list',
app_labels=app_labels,
database=db,
no_color=options.get('no_color', False),
settings=options.get('settings', os.environ['DJANGO_SETTINGS_MODULE']),
stdout=out,
traceback=options.get('traceback', True),
verbosity=verbosity,
)
connections.close_all()
out.seek(0)
output = out.read()
return re.sub('\x1b\[(1|0)m', '', output)
def auto_discover():
'''
Auto register all apps that have module moderator with moderation
'''
from django.apps import apps
for app in apps.get_app_configs():
import_moderator(app.name)
def get_models(app_labels):
"""
Get a list of models for the given app labels, with some exceptions.
TODO: If a required model is referenced, it should also be included.
Or at least discovered with a get_or_create() call.
"""
# These models are not to be output, e.g. because they can be generated automatically
# TODO: This should be "appname.modelname" string
EXCLUDED_MODELS = (ContentType, )
models = []
# If no app labels are given, return all
if not app_labels:
for app in apps.get_app_configs():
models += [m for m in apps.get_app_config(app.label).get_models()
if m not in EXCLUDED_MODELS]
return models
# Get all relevant apps
for app_label in app_labels:
# If a specific model is mentioned, get only that model
if "." in app_label:
app_label, model_name = app_label.split(".", 1)
models.append(apps.get_model(app_label, model_name))
# Get all models for a given app
else:
models += [m for m in apps.get_app_config(app_label).get_models()
if m not in EXCLUDED_MODELS]
return models
def load_disk(self):
"""Load the migrations from all INSTALLED_APPS from disk."""
self.disk_migrations = {}
self.unmigrated_apps = set()
self.migrated_apps = set()
for app_config in apps.get_app_configs():
# Get the migrations module directory
module_name, explicit = self.migrations_module(app_config.label)
if module_name is None:
self.unmigrated_apps.add(app_config.label)
continue
was_loaded = module_name in sys.modules
try:
module = import_module(module_name)
except ImportError as e:
# I hate doing this, but I don't want to squash other import errors.
# Might be better to try a directory check directly.
if ((explicit and self.ignore_no_migrations)
or (not explicit and "No module named" in str(e)
and MIGRATIONS_MODULE_NAME in str(e))):
self.unmigrated_apps.add(app_config.label)
continue
raise
else:
# Empty directories are namespaces.
# getattr() needed on PY36 and older (replace w/attribute access).
if getattr(module, '__file__', None) is None:
self.unmigrated_apps.add(app_config.label)
continue
# Module is not a package (e.g. migrations.py).
if not hasattr(module, '__path__'):
self.unmigrated_apps.add(app_config.label)
continue
# Force a reload if it's already loaded (tests need this)
if was_loaded:
reload(module)
self.migrated_apps.add(app_config.label)
migration_names = {
name
for _, name, is_pkg in pkgutil.iter_modules(module.__path__)
if not is_pkg and name[0] not in '_~'
}
# Load migrations
for migration_name in migration_names:
migration_path = '%s.%s' % (module_name, migration_name)
try:
migration_module = import_module(migration_path)
except ImportError as e:
if 'bad magic number' in str(e):
raise ImportError(
"Couldn't import %r as it appears to be a stale "
".pyc file." % migration_path) from e
else:
raise
if not hasattr(migration_module, "Migration"):
raise BadMigrationError(
"Migration %s in app %s has no Migration class" %
(migration_name, app_config.label))
self.disk_migrations[
app_config.label,
migration_name] = migration_module.Migration(
migration_name,
app_config.label,
)
def _get_all_configs():
return [
app for app in apps.get_app_configs()
if app.name in settings.INSTALLED_OTREE_APPS
]
# coding: utf-8
from __future__ import absolute_import
import os
from django.apps import apps
from celery import Celery
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "commitz.settings.local")
app = Celery('commitz_tasks')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()])
def handle(self, *args, **options):
self.verbosity = options['verbosity']
self.interactive = options['interactive']
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_config in apps.get_app_configs():
if module_has_submodule(app_config.module, "management"):
import_module('.management', app_config.name)
# Get the database we're operating from
db = options['database']
connection = connections[db]
# Hook for backends needing any database preparation
connection.prepare_database()
# Work out which apps have migrations and which do not
executor = MigrationExecutor(connection,
self.migration_progress_callback)
# Raise an error if any migrations are applied before their dependencies.
executor.loader.check_consistent_history(connection)
# Before anything else, see if there's conflicting apps and drop out
# hard if there are any
conflicts = executor.loader.detect_conflicts()
if conflicts:
name_str = "; ".join("%s in %s" % (", ".join(names), app)
for app, names in conflicts.items())
raise CommandError(
"Conflicting migrations detected; multiple leaf nodes in the "
"migration graph: (%s).\nTo fix them run "
"'python manage.py makemigrations --merge'" % name_str)
# If they supplied command line arguments, work out what they mean.
target_app_labels_only = True
if options['app_label'] and options['migration_name']:
app_label, migration_name = options['app_label'], options[
'migration_name']
if app_label not in executor.loader.migrated_apps:
raise CommandError("App '%s' does not have migrations." %
app_label)
if migration_name == "zero":
targets = [(app_label, None)]
else:
try:
migration = executor.loader.get_migration_by_prefix(
app_label, migration_name)
except AmbiguityError:
raise CommandError(
"More than one migration matches '%s' in app '%s'. "
"Please be more specific." %
(migration_name, app_label))
except KeyError:
raise CommandError(
"Cannot find a migration matching '%s' from app '%s'."
% (migration_name, app_label))
targets = [(app_label, migration.name)]
target_app_labels_only = False
elif options['app_label']:
app_label = options['app_label']
if app_label not in executor.loader.migrated_apps:
raise CommandError("App '%s' does not have migrations." %
app_label)
targets = [
key for key in executor.loader.graph.leaf_nodes()
if key[0] == app_label
]
else:
targets = executor.loader.graph.leaf_nodes()
plan = executor.migration_plan(targets)
run_syncdb = options['run_syncdb'] and executor.loader.unmigrated_apps
# Print some useful info
if self.verbosity >= 1:
self.stdout.write(
self.style.MIGRATE_HEADING("Operations to perform:"))
if run_syncdb:
self.stdout.write(
self.style.MIGRATE_LABEL(" Synchronize unmigrated apps: ")
+ (", ".join(sorted(executor.loader.unmigrated_apps))))
if target_app_labels_only:
self.stdout.write(
self.style.MIGRATE_LABEL(" Apply all migrations: ") +
(", ".join(sorted(set(a
for a, n in targets))) or "(none)"))
else:
if targets[0][1] is None:
self.stdout.write(
self.style.MIGRATE_LABEL(" Unapply all migrations: ")
+ "%s" % (targets[0][0], ))
else:
self.stdout.write(
self.style.MIGRATE_LABEL(
" Target specific migration: ") + "%s, from %s" %
(targets[0][1], targets[0][0]))
pre_migrate_state = executor._create_project_state(
with_applied_migrations=True)
pre_migrate_apps = pre_migrate_state.apps
emit_pre_migrate_signal(
self.verbosity,
self.interactive,
connection.alias,
apps=pre_migrate_apps,
plan=plan,
)
# Run the syncdb phase.
if run_syncdb:
if self.verbosity >= 1:
self.stdout.write(
self.style.MIGRATE_HEADING(
"Synchronizing apps without migrations:"))
self.sync_apps(connection, executor.loader.unmigrated_apps)
# Migrate!
if self.verbosity >= 1:
self.stdout.write(
self.style.MIGRATE_HEADING("Running migrations:"))
if not plan:
if self.verbosity >= 1:
self.stdout.write(" No migrations to apply.")
# If there's changes that aren't in migrations yet, tell them how to fix it.
autodetector = MigrationAutodetector(
executor.loader.project_state(),
ProjectState.from_apps(apps),
)
changes = autodetector.changes(graph=executor.loader.graph)
if changes:
self.stdout.write(
self.style.NOTICE(
" Your models have changes that are not yet reflected "
"in a migration, and so won't be applied."))
self.stdout.write(
self.style.NOTICE(
" Run 'manage.py makemigrations' to make new "
"migrations, and then re-run 'manage.py migrate' to "
"apply them."))
fake = False
fake_initial = False
else:
fake = options['fake']
fake_initial = options['fake_initial']
post_migrate_state = executor.migrate(
targets,
plan=plan,
state=pre_migrate_state.clone(),
fake=fake,
fake_initial=fake_initial,
)
# post_migrate signals have access to all models. Ensure that all models
# are reloaded in case any are delayed.
post_migrate_state.clear_delayed_apps_cache()
post_migrate_apps = post_migrate_state.apps
# Re-render models of real apps to include relationships now that
# we've got a final state. This wouldn't be necessary if real apps
# models were rendered with relationships in the first place.
with post_migrate_apps.bulk_update():
model_keys = []
for model_state in post_migrate_apps.real_models:
model_key = model_state.app_label, model_state.name_lower
model_keys.append(model_key)
post_migrate_apps.unregister_model(*model_key)
post_migrate_apps.render_multiple([
ModelState.from_model(apps.get_model(*model))
for model in model_keys
])
# Send the post_migrate signal, so individual apps can do whatever they need
# to do at this point.
emit_post_migrate_signal(
self.verbosity,
self.interactive,
connection.alias,
apps=post_migrate_apps,
plan=plan,
)
def test_get_app_configs(self):
"""
Tests apps.get_app_configs().
"""
app_configs = apps.get_app_configs()
self.assertEqual([app_config.name for app_config in app_configs], SOME_INSTALLED_APPS_NAMES)
def get_app_by_name(name):
all_apps = apps.get_app_configs()
participation_apps = [a for a in all_apps if a.name==name and len(get_app_project_models(a))>=1]
return participation_apps[0]
def get_registered_participation_apps():
all_apps = apps.get_app_configs()
participation_apps = [a for a in all_apps if not a.name=="core" and len(get_app_project_models(a))>=1]
return participation_apps
def get_core_app():
all_apps = apps.get_app_configs()
return [a for a in all_apps if a.name=="core" and len(get_app_project_models(a))==1][0]
def autodiscover_tasks(self):
from django.apps import apps
return [config.name for config in apps.get_app_configs()]
def render(self, name, value, attrs=None, choices=()):
apps_available = [] # main container to send to template
user_permissions = Permission.objects.filter(
id__in=value or []).values_list('id', flat=True)
all_perms = Permission.objects.all().values(
'id', 'codename', 'content_type_id').order_by('codename')
excluded_perms = set([])
codename_id_map = {}
for p in all_perms:
codename_id_map['%s_%s' %
(p['codename'], p['content_type_id'])] = p['id']
reminder_perms = codename_id_map.copy()
# used to detect if the tabular permissions covers all permissions, if so, we don't need to make it visible.
for app in apps.get_app_configs():
app_dict = {
'verbose_name': force_text(app.verbose_name),
'models': []
}
for model_name in app.models:
model = app.models[model_name]
ct_id = ContentType.objects.get_for_model(
model,
for_concrete_model=TABULAR_PERMISSIONS_USE_FOR_CONCRETE).pk
add_perm_name = get_perm_name(model_name, 'add')
change_perm_name = get_perm_name(model_name, 'change')
delete_perm_name = get_perm_name(model_name, 'delete')
# pdb.set_trace()
add_perm_id = codename_id_map.get(
'%s_%s' % (add_perm_name, ct_id), False)
change_perm_id = codename_id_map.get(
'%s_%s' % (change_perm_name, ct_id), False)
delete_perm_id = codename_id_map.get(
'%s_%s' % (delete_perm_name, ct_id), False)
if add_perm_id and change_perm_id and delete_perm_id and not {
add_perm_id, change_perm_id, delete_perm_id
} & excluded_perms:
excluded_perms.update(
[add_perm_id, change_perm_id, delete_perm_id])
reminder_perms.pop('%s_%s' % (add_perm_name, ct_id))
reminder_perms.pop('%s_%s' % (change_perm_name, ct_id))
reminder_perms.pop('%s_%s' % (delete_perm_name, ct_id))
if app.label in TABULAR_PERMISSIONS_EXCLUDE_APPS \
or model_name in TABULAR_PERMISSIONS_EXCLUDE_MODELS \
or TABULAR_PERMISSIONS_EXCLUDE_FUNCTION(model):
continue
app_dict['models'].append({
'model_name':
model_name,
'model':
model,
'verbose_name_plural':
force_text(model._meta.verbose_name_plural),
'verbose_name':
force_text(model._meta.verbose_name),
'add_perm_id':
add_perm_id,
'add_perm_name':
add_perm_name,
'change_perm_id':
change_perm_id,
'change_perm_name':
change_perm_name,
'delete_perm_id':
delete_perm_id,
'delete_perm_name':
delete_perm_name,
})
if app.models:
apps_available.append(app_dict)
request_context = {
'apps_available': apps_available,
'user_permissions': user_permissions,
'codename_id_map': codename_id_map,
'input_name': self.input_name
}
body = get_template(TABULAR_PERMISSIONS_TEMPLATE).render(
request_context).encode("utf-8")
self.managed_perms = excluded_perms
if reminder_perms:
self.hide_original = False
# Get "original" FilteredSelectMultiple, and hide it if necessary.
# Next block is a "copy" of FilteredSelectMultiple render(), except the if reminder_perms: check.
# Due to change in how SelectFilter take its arguments and the dropping of static('admin/') in django1.9
# there a check on django version
if attrs is None:
attrs = {}
if '1.10' in django_version:
attrs['class'] = 'selector-filter'
else:
attrs['class'] = 'selectfilter'
if self.is_stacked:
attrs['class'] += 'stacked'
output = [
super(FilteredSelectMultiple, self).render(name, value, attrs)
]
if reminder_perms:
output.append(
'<script type="text/javascript">addEvent(window, "load", function(e) {'
)
if '1.8' in django_version:
output.append(
'SelectFilter.init("id_%s", "%s", %s, "%s"); });</script>\n'
% (name, self.verbose_name.replace(
'"', '\\"'), int(self.is_stacked), static('admin/')))
else: # 1.9
output.append(
'SelectFilter.init("id_%s", "%s", %s); });</script>\n' %
(name, escapejs(self.verbose_name), int(self.is_stacked)))
initial = mark_safe(''.join(output))
response = ' <hr/>'.join([force_text(body), force_text(initial)])
return mark_safe(response)
checkinlist_router.register(r'positions', checkin.CheckinListPositionViewSet)
question_router = routers.DefaultRouter()
question_router.register(r'options', item.QuestionOptionViewSet)
item_router = routers.DefaultRouter()
item_router.register(r'variations', item.ItemVariationViewSet)
item_router.register(r'addons', item.ItemAddOnViewSet)
item_router.register(r'bundles', item.ItemBundleViewSet)
order_router = routers.DefaultRouter()
order_router.register(r'payments', order.PaymentViewSet)
order_router.register(r'refunds', order.RefundViewSet)
# Force import of all plugins to give them a chance to register URLs with the router
for app in apps.get_app_configs():
if hasattr(app, 'PretixPluginMeta'):
if importlib.util.find_spec(app.name + '.urls'):
importlib.import_module(app.name + '.urls')
urlpatterns = [
url(r'^', include(router.urls)),
url(r'^organizers/(?P<organizer>[^/]+)/', include(orga_router.urls)),
url(r'^organizers/(?P<organizer>[^/]+)/events/(?P<event>[^/]+)/',
include(event_router.urls)),
url(
r'^organizers/(?P<organizer>[^/]+)/events/(?P<event>[^/]+)/items/(?P<item>[^/]+)/',
include(item_router.urls)),
url(
r'^organizers/(?P<organizer>[^/]+)/events/(?P<event>[^/]+)/questions/(?P<question>[^/]+)/',
include(question_router.urls)),
def load_stores_from_fixture(apps, schema_editor):
for app_config in django_apps.get_app_configs():
update_contenttypes(app_config)
call_command("loaddata", "0003_data.json")
def ready(self):
installed_apps = [
app_config.name for app_config in apps.get_app_configs()
]
app.autodiscover_tasks(lambda: installed_apps, force=True)
def init_cache(self, realm: str) -> None:
angular_modules: List[str] = []
js_files: List[str] = []
for app_config in apps.get_app_configs():
# Add the angular app if the module has one.
if getattr(app_config, 'angular_{}_module'.format(realm), False):
angular_modules.append(
'OpenSlidesApp.{app_name}.{realm}'.format(
app_name=app_config.label, realm=realm))
# Add all JavaScript files that the module needs. Our core apps
# are delivered by an extra file js/openslides.js which can be
# created via gulp.
core_apps = (
'openslides.core',
'openslides.agenda',
'openslides.motions',
'openslides.assignments',
'openslides.users',
'openslides.mediafiles',
)
if app_config.name not in core_apps:
try:
app_js_files = app_config.js_files
except AttributeError:
# The app needs no JavaScript files.
pass
else:
js_files.extend(app_js_files)
# angular constants
angular_constants = ''
for key, value in get_constants().items():
value = json.dumps(value)
angular_constants += ".constant('{}', {})".format(key, value)
# Use JavaScript loadScript function from
# http://balpha.de/2011/10/jquery-script-insertion-and-its-consequences-for-debugging/
# jQuery is required.
content = dedent("""
(function () {
var loadScript = function (path) {
var result = $.Deferred(),
script = document.createElement("script");
script.async = "async";
script.type = "text/javascript";
script.src = path;
script.onload = script.onreadystatechange = function(_, isAbort) {
if (!script.readyState || /loaded|complete/.test(script.readyState)) {
if (isAbort)
result.reject();
else
result.resolve();
}
};
script.onerror = function () { result.reject(); };
$("head")[0].appendChild(script);
return result.promise();
};
""" + """
angular.module('OpenSlidesApp.{realm}', {angular_modules}){angular_constants};
var deferres = [];
{js_files}.forEach( function(js_file) {{ deferres.push(loadScript(js_file)); }} );
$.when.apply(this,deferres).done( function() {{
angular.bootstrap(document,['OpenSlidesApp.{realm}']);
}} );
""".format(realm=realm,
angular_modules=angular_modules,
angular_constants=angular_constants,
js_files=js_files) + """
}());
""").replace('\n', '')
self.cache[realm] = content
def generate_graph_data(app_labels, **kwargs):
cli_options = kwargs.get('cli_options', None)
disable_fields = kwargs.get('disable_fields', False)
include_models = parse_file_or_list(kwargs.get('include_models', ""))
all_applications = kwargs.get('all_applications', False)
use_subgraph = kwargs.get('group_models', False)
verbose_names = kwargs.get('verbose_names', False)
inheritance = kwargs.get('inheritance', True)
relations_as_fields = kwargs.get("relations_as_fields", True)
sort_fields = kwargs.get("sort_fields", True)
language = kwargs.get('language', None)
if language is not None:
activate_language(language)
exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', ""))
exclude_models = parse_file_or_list(kwargs.get('exclude_models', ""))
def skip_field(field):
if exclude_columns:
if verbose_names and field.verbose_name:
if field.verbose_name in exclude_columns:
return True
if field.name in exclude_columns:
return True
return False
if all_applications:
app_labels = [app.label for app in apps.get_app_configs()]
graphs = []
for app_label in app_labels:
app = apps.get_app_config(app_label)
if not app:
continue
graph = Context({
'name': '"%s"' % app.name,
'app_name': "%s" % app.name,
'cluster_app_name': "cluster_%s" % app.name.replace(".", "_"),
'models': []
})
appmodels = list(app.get_models())
abstract_models = []
for appmodel in appmodels:
abstract_models = abstract_models + [abstract_model for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract]
abstract_models = list(set(abstract_models)) # remove duplicates
appmodels = abstract_models + appmodels
for appmodel in appmodels:
appmodel_abstracts = [abstract_model.__name__ for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract]
# collect all attribs of abstract superclasses
def getBasesAbstractFields(c):
_abstract_fields = []
for e in c.__bases__:
if hasattr(e, '_meta') and e._meta.abstract:
_abstract_fields.extend(e._meta.fields)
_abstract_fields.extend(getBasesAbstractFields(e))
return _abstract_fields
abstract_fields = getBasesAbstractFields(appmodel)
model = {
'app_name': appmodel.__module__.replace(".", "_"),
'name': appmodel.__name__,
'abstracts': appmodel_abstracts,
'fields': [],
'relations': []
}
if not use_model(
appmodel._meta.object_name,
include_models,
exclude_models
):
continue
if verbose_names and appmodel._meta.verbose_name:
model['label'] = force_bytes(appmodel._meta.verbose_name)
else:
model['label'] = model['name']
# model attributes
def add_attributes(field):
if verbose_names and field.verbose_name:
label = force_bytes(field.verbose_name)
if label.islower():
label = label.capitalize()
else:
label = field.name
t = type(field).__name__
if isinstance(field, (OneToOneField, ForeignKey)):
t += " ({0})".format(field.rel.field_name)
# TODO: ManyToManyField, GenericRelation
model['fields'].append({
'name': field.name,
'label': label,
'type': t,
'blank': field.blank,
'abstract': field in abstract_fields,
'relation': isinstance(field, RelatedField),
'primary_key': field.primary_key,
})
attributes = [field for field in appmodel._meta.local_fields]
if not relations_as_fields:
# Find all the 'real' attributes. Relations are depicted as graph edges instead of attributes
attributes = [field for field in attributes if not isinstance(field, RelatedField)]
# find primary key and print it first, ignoring implicit id if other pk exists
pk = appmodel._meta.pk
if pk and not appmodel._meta.abstract and pk in attributes:
add_attributes(pk)
for field in attributes:
if skip_field(field):
continue
if pk and field == pk:
continue
add_attributes(field)
if sort_fields:
model['fields'] = sorted(model['fields'], key=lambda field: (not field['primary_key'], not field['relation'], field['label']))
# FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph?
# if appmodel._meta.many_to_many:
# for field in appmodel._meta.many_to_many:
# if skip_field(field):
# continue
# add_attributes(field)
# relations
def add_relation(field, extras=""):
if verbose_names and field.verbose_name:
label = force_bytes(field.verbose_name)
if label.islower():
label = label.capitalize()
else:
label = field.name
# show related field name
if hasattr(field, 'related_query_name'):
related_query_name = field.related_query_name()
if verbose_names and related_query_name.islower():
related_query_name = related_query_name.replace('_', ' ').capitalize()
label = '{} ({})'.format(label, force_bytes(related_query_name))
# handle self-relationships and lazy-relationships
if isinstance(field.rel.to, six.string_types):
if field.rel.to == 'self':
target_model = field.model
else:
if '.' in field.rel.to:
app_label, model_name = field.rel.to.split('.', 1)
else:
app_label = field.model._meta.app_label
model_name = field.rel.to
target_model = apps.get_model(app_label, model_name)
else:
target_model = field.rel.to
_rel = {
'target_app': target_model.__module__.replace('.', '_'),
'target': target_model.__name__,
'type': type(field).__name__,
'name': field.name,
'label': label,
'arrows': extras,
'needs_node': True
}
if _rel not in model['relations'] and use_model(
_rel['target'],
include_models,
exclude_models
):
model['relations'].append(_rel)
for field in appmodel._meta.local_fields:
if field.attname.endswith('_ptr_id'): # excluding field redundant with inheritance relation
continue
if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields
continue
if skip_field(field):
continue
if isinstance(field, OneToOneField):
add_relation(field, '[arrowhead=none, arrowtail=none, dir=both]')
elif isinstance(field, ForeignKey):
add_relation(field, '[arrowhead=none, arrowtail=dot, dir=both]')
for field in appmodel._meta.local_many_to_many:
if skip_field(field):
continue
if isinstance(field, ManyToManyField):
if hasattr(field.rel.through, '_meta') and field.rel.through._meta.auto_created:
add_relation(field, '[arrowhead=dot arrowtail=dot, dir=both]')
elif isinstance(field, GenericRelation):
add_relation(field, mark_safe('[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]'))
if inheritance:
# add inheritance arrows
for parent in appmodel.__bases__:
if hasattr(parent, "_meta"): # parent is a model
l = "multi-table"
if parent._meta.abstract:
l = "abstract"
if appmodel._meta.proxy:
l = "proxy"
l += r"\ninheritance"
_rel = {
'target_app': parent.__module__.replace(".", "_"),
'target': parent.__name__,
'type': "inheritance",
'name': "inheritance",
'label': l,
'arrows': '[arrowhead=empty, arrowtail=none, dir=both]',
'needs_node': True,
}
# TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes.
if _rel not in model['relations'] and use_model(
_rel['target'],
include_models,
exclude_columns
):
model['relations'].append(_rel)
graph['models'].append(model)
if graph['models']:
graphs.append(graph)
nodes = []
for graph in graphs:
nodes.extend([e['name'] for e in graph['models']])
for graph in graphs:
for model in graph['models']:
for relation in model['relations']:
if relation['target'] in nodes:
relation['needs_node'] = False
now = datetime.datetime.now()
graph_data = {
'created_at': now.strftime("%Y-%m-%d %H:%M"),
'cli_options': cli_options,
'disable_fields': disable_fields,
'use_subgraph': use_subgraph,
'graphs': graphs,
}
return graph_data
def autocomplete(self):
"""
Output completion suggestions for BASH.
The output of this function is passed to BASH's `COMREPLY` variable and
treated as completion suggestions. `COMREPLY` expects a space
separated string as the result.
The `COMP_WORDS` and `COMP_CWORD` BASH environment variables are used
to get information about the cli input. Please refer to the BASH
man-page for more information about this variables.
Subcommand options are saved as pairs. A pair consists of
the long option string (e.g. '--exclude') and a boolean
value indicating if the option requires arguments. When printing to
stdout, an equal sign is appended to options which require arguments.
Note: If debugging this function, it is recommended to write the debug
output in a separate file. Otherwise the debug output will be treated
and formatted as potential completion suggestions.
"""
# Don't complete if user hasn't sourced bash_completion file.
if 'DJANGO_AUTO_COMPLETE' not in os.environ:
return
cwords = os.environ['COMP_WORDS'].split()[1:]
cword = int(os.environ['COMP_CWORD'])
try:
curr = cwords[cword - 1]
except IndexError:
curr = ''
subcommands = list(get_commands()) + ['help']
options = [('--help', False)]
# subcommand
if cword == 1:
print(' '.join(sorted(filter(lambda x: x.startswith(curr), subcommands))))
# subcommand options
# special case: the 'help' subcommand has no options
elif cwords[0] in subcommands and cwords[0] != 'help':
subcommand_cls = self.fetch_command(cwords[0])
# special case: add the names of installed apps to options
if cwords[0] in ('dumpdata', 'sqlmigrate', 'sqlsequencereset', 'test'):
try:
app_configs = apps.get_app_configs()
# Get the last part of the dotted path as the app name.
options.extend((app_config.label, 0) for app_config in app_configs)
except ImportError:
# Fail silently if DJANGO_SETTINGS_MODULE isn't set. The
# user will find out once they execute the command.
pass
parser = subcommand_cls.create_parser('', cwords[0])
options.extend(
(min(s_opt.option_strings), s_opt.nargs != 0)
for s_opt in parser._actions if s_opt.option_strings
)
# filter out previously specified options from available options
prev_opts = {x.split('=')[0] for x in cwords[1:cword - 1]}
options = (opt for opt in options if opt[0] not in prev_opts)
# filter options by current input
options = sorted((k, v) for k, v in options if k.startswith(curr))
for opt_label, require_arg in options:
# append '=' to options which require args
if require_arg:
opt_label += '='
print(opt_label)
# Exit code of the bash completion function is never passed back to
# the user, so it's safe to always exit with 0.
# For more details see #25420.
sys.exit(0)
def find_pos(lang,
project_apps=True,
django_apps=False,
third_party_apps=False):
"""
scans a couple possible repositories of gettext catalogs for the given
language code
"""
paths = []
# project/locale
parts = settings.SETTINGS_MODULE.split('.')
project = __import__(parts[0], {}, {}, [])
abs_project_path = os.path.normpath(
os.path.abspath(os.path.dirname(project.__file__)))
if project_apps:
if os.path.exists(
os.path.abspath(
os.path.join(os.path.dirname(project.__file__),
'locale'))):
paths.append(
os.path.abspath(
os.path.join(os.path.dirname(project.__file__), 'locale')))
if os.path.exists(
os.path.abspath(
os.path.join(os.path.dirname(project.__file__), '..',
'locale'))):
paths.append(
os.path.abspath(
os.path.join(os.path.dirname(project.__file__), '..',
'locale')))
# django/locale
if django_apps:
django_paths = cache.get('rosetta_django_paths')
if django_paths is None:
django_paths = []
for root, dirnames, filename in os.walk(
os.path.abspath(os.path.dirname(django.__file__))):
if 'locale' in dirnames:
django_paths.append(os.path.join(root, 'locale'))
continue
cache.set('rosetta_django_paths', django_paths, 60 * 60)
paths = paths + django_paths
# settings
for localepath in settings.LOCALE_PATHS:
if os.path.isdir(localepath):
paths.append(localepath)
# project/app/locale
for app_ in apps.get_app_configs():
if rosetta_settings.EXCLUDED_APPLICATIONS and app_.name in rosetta_settings.EXCLUDED_APPLICATIONS:
continue
app_path = app_.path
# django apps
if 'contrib' in app_path and 'django' in app_path and not django_apps:
continue
# third party external
if not third_party_apps and abs_project_path not in app_path:
continue
# local apps
if not project_apps and abs_project_path in app_path:
continue
if os.path.exists(os.path.abspath(os.path.join(app_path, 'locale'))):
paths.append(os.path.abspath(os.path.join(app_path, 'locale')))
if os.path.exists(
os.path.abspath(os.path.join(app_path, '..', 'locale'))):
paths.append(
os.path.abspath(os.path.join(app_path, '..', 'locale')))
ret = set()
langs = [lang]
if u'-' in lang:
_l, _c = map(lambda x: x.lower(), lang.split(u'-', 1))
langs += [
u'%s_%s' % (_l, _c),
u'%s_%s' % (_l, _c.upper()),
u'%s_%s' % (_l, _c.capitalize())
]
elif u'_' in lang:
_l, _c = map(lambda x: x.lower(), lang.split(u'_', 1))
langs += [
u'%s-%s' % (_l, _c),
u'%s-%s' % (_l, _c.upper()),
u'%s_%s' % (_l, _c.capitalize())
]
paths = map(os.path.normpath, paths)
paths = list(set(paths))
for path in paths:
# Exclude paths
if path not in rosetta_settings.ROSETTA_EXCLUDED_PATHS:
for lang_ in langs:
dirname = os.path.join(path, lang_, 'LC_MESSAGES')
for fn in rosetta_settings.POFILENAMES:
filename = os.path.join(dirname, fn)
if os.path.isfile(filename):
ret.add(os.path.abspath(filename))
return list(sorted(ret))
def handle(self, *app_labels, **options):
self.verbosity = options['verbosity']
self.interactive = options['interactive']
self.dry_run = options['dry_run']
self.merge = options['merge']
self.empty = options['empty']
self.migration_name = options['name']
if self.migration_name and not self.migration_name.isidentifier():
raise CommandError('The migration name must be a valid Python identifier.')
check_changes = options['check_changes']
# Make sure the app they asked for exists
app_labels = set(app_labels)
has_bad_labels = False
for app_label in app_labels:
try:
apps.get_app_config(app_label)
except LookupError as err:
self.stderr.write(str(err))
has_bad_labels = True
if has_bad_labels:
sys.exit(2)
# Load the current graph state. Pass in None for the connection so
# the loader doesn't try to resolve replaced migrations from DB.
loader = MigrationLoader(None, ignore_no_migrations=True)
# Raise an error if any migrations are applied before their dependencies.
consistency_check_labels = {config.label for config in apps.get_app_configs()}
# Non-default databases are only checked if database routers used.
aliases_to_check = connections if settings.DATABASE_ROUTERS else [DEFAULT_DB_ALIAS]
for alias in sorted(aliases_to_check):
connection = connections[alias]
if (connection.settings_dict['ENGINE'] != 'django.db.backends.dummy' and any(
# At least one model must be migrated to the database.
router.allow_migrate(connection.alias, app_label, model_name=model._meta.object_name)
for app_label in consistency_check_labels
for model in apps.get_app_config(app_label).get_models()
)):
loader.check_consistent_history(connection)
# Before anything else, see if there's conflicting apps and drop out
# hard if there are any and they don't want to merge
conflicts = loader.detect_conflicts()
# If app_labels is specified, filter out conflicting migrations for unspecified apps
if app_labels:
conflicts = {
app_label: conflict for app_label, conflict in conflicts.items()
if app_label in app_labels
}
if conflicts and not self.merge:
name_str = "; ".join(
"%s in %s" % (", ".join(names), app)
for app, names in conflicts.items()
)
raise CommandError(
"Conflicting migrations detected; multiple leaf nodes in the "
"migration graph: (%s).\nTo fix them run "
"'python manage.py makemigrations --merge'" % name_str
)
# If they want to merge and there's nothing to merge, then politely exit
if self.merge and not conflicts:
self.stdout.write("No conflicts detected to merge.")
return
# If they want to merge and there is something to merge, then
# divert into the merge code
if self.merge and conflicts:
return self.handle_merge(loader, conflicts)
if self.interactive:
questioner = InteractiveMigrationQuestioner(specified_apps=app_labels, dry_run=self.dry_run)
else:
questioner = NonInteractiveMigrationQuestioner(specified_apps=app_labels, dry_run=self.dry_run)
# Set up autodetector
autodetector = MigrationAutodetector(
loader.project_state(),
ProjectState.from_apps(apps),
questioner,
)
# If they want to make an empty migration, make one for each app
if self.empty:
if not app_labels:
raise CommandError("You must supply at least one app label when using --empty.")
# Make a fake changes() result we can pass to arrange_for_graph
changes = {
app: [Migration("custom", app)]
for app in app_labels
}
changes = autodetector.arrange_for_graph(
changes=changes,
graph=loader.graph,
migration_name=self.migration_name,
)
self.write_migration_files(changes)
return
# Detect changes
changes = autodetector.changes(
graph=loader.graph,
trim_to_apps=app_labels or None,
convert_apps=app_labels or None,
migration_name=self.migration_name,
)
if not changes:
# No changes? Tell them.
if self.verbosity >= 1:
if app_labels:
if len(app_labels) == 1:
self.stdout.write("No changes detected in app '%s'" % app_labels.pop())
else:
self.stdout.write("No changes detected in apps '%s'" % ("', '".join(app_labels)))
else:
self.stdout.write("No changes detected")
else:
self.write_migration_files(changes)
if check_changes:
sys.exit(1)
def handle(self, *app_labels, **options):
# Activate project's default language
translation.activate(settings.LANGUAGE_CODE)
comment = options["comment"]
batch_size = options["batch_size"]
database = options.get('database')
verbosity = int(options.get("verbosity", 1))
app_list = OrderedDict()
# if no apps given, use all installed.
if len(app_labels) == 0:
all_apps = [
config.models_module for config in apps.get_app_configs()
if config.models_module is not None
]
for app in all_apps:
if not app in app_list:
app_list[app] = []
for model_class in apps.get_models(app):
if not model_class in app_list[app]:
app_list[app].append(model_class)
else:
for label in app_labels:
try:
app_label, model_label = label.split(".")
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" %
app_label)
model_class = apps.get_model(app_label, model_label)
if model_class is None:
raise CommandError("Unknown model: %s.%s" %
(app_label, model_label))
if app in app_list:
if app_list[app] and model_class not in app_list[app]:
app_list[app].append(model_class)
else:
app_list[app] = [model_class]
except ValueError:
# This is just an app - no model qualifier.
app_label = label
try:
app = get_app(app_label)
if not app in app_list:
app_list[app] = []
for model_class in apps.get_models(app):
if not model_class in app_list[app]:
app_list[app].append(model_class)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" %
app_label)
# Create revisions.
for app, model_classes in app_list.items():
for model_class in model_classes:
self.create_initial_revisions(app,
model_class,
comment,
batch_size,
verbosity,
database=database)
# Go back to default language
translation.deactivate()
def ready(self):
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
installed_apps = [app_config.name for app_config in apps.get_app_configs()]
app.autodiscover_tasks(lambda: installed_apps, force=True)
def load_initial_objects(self):
# look for any .xml files in apps under fixtures/initial_objects
# and attempt to load them as Fedora objects
# NOTE! any fixtures should have pids specified, or new versions of the
# fixture will be created every time syncrepo runs
app_module_paths = []
if hasattr(django_apps, 'get_app_configs'):
apps = django_apps.get_app_configs()
else:
apps = get_apps()
# monkey see django code, monkey do
for app in apps:
# newer django AppConfig
if hasattr(app, 'path'):
app_module_paths.append(app.path)
elif hasattr(app, '__path__'):
# It's a 'models/' subpackage
for path in app.__path__:
app_module_paths.append(path)
else:
# It's a models.py module
app_module_paths.append(app.__file__)
app_fixture_paths = [
os.path.join(os.path.dirname(path), 'fixtures', 'initial_objects',
'*.xml') for path in app_module_paths
]
fixture_count = 0
load_count = 0
for path in app_fixture_paths:
fixtures = glob.iglob(path)
for f in fixtures:
# FIXME: is there a sane, sensible way to shorten file path for error/success messages?
fixture_count += 1
with open(f) as fixture_data:
# rather than pulling PID from fixture and checking if it already exists,
# just ingest and catch appropriate excetions
try:
pid = self.repo.ingest(fixture_data.read(),
"loaded from fixture")
if self.verbosity > 1:
self.stdout.write("Loaded fixture %s as %s" %
(f, pid))
load_count += 1
except RequestFailed as rf:
if hasattr(rf, 'detail'):
if 'ObjectExistsException' in rf.detail or \
'already exists in the registry; the object can\'t be re-created' in rf.detail:
if self.verbosity > 1:
self.stdout.write(
"Fixture %s has already been loaded" %
f)
elif 'ObjectValidityException' in rf.detail:
# could also look for: fedora.server.errors.ValidationException
# (e.g., RELS-EXT about does not match pid)
self.stdout.write(
"Error: fixture %s is not a valid Repository object"
% f)
else:
# if there is at least a detail message, display that
self.stdout.write("Error ingesting %s: %s" %
(f, rf.detail))
else:
raise rf
# summarize what was actually done
if self.verbosity > 0:
if fixture_count == 0:
self.stdout.write("No fixtures found")
else:
self.stdout.write("Loaded %d object(s) from %d fixture(s)" %
(load_count, fixture_count))
def get_installed():
return [app_config.name for app_config in apps.get_app_configs()]
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config,
connection.alias,
include_auto_created=True))
for app_config in apps.get_app_configs()
if app_config.models_module is not None
and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
create_models = set(itertools.chain(*manifest.values()))
emit_pre_migrate_signal(create_models, self.verbosity,
self.interactive, connection.alias)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=False):
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" %
(app_name, model._meta.object_name))
sql, references = connection.creation.sql_create_model(
model, no_style(), seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto,
[]).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.
sql_for_pending_references(
refto, no_style(), pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, no_style(), pending_references))
if self.verbosity >= 1 and sql:
self.stdout.write(" Creating table %s\n" %
model._meta.db_table)
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
# We force a commit here, as that was the previous behavior.
# If you can prove we don't need this, remove it.
transaction.set_dirty(using=connection.alias)
finally:
cursor.close()
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
try:
# Install custom SQL for the app (but only if this
# is a model we've just created)
if self.verbosity >= 1:
self.stdout.write(" Installing custom SQL...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(
model, no_style(), connection)
if custom_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing custom SQL for %s.%s model\n"
% (app_name, model._meta.object_name))
try:
with transaction.commit_on_success_unless_managed(
using=connection.alias):
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install custom SQL for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e))
if self.show_traceback:
traceback.print_exc()
else:
if self.verbosity >= 3:
self.stdout.write(
" No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
if self.verbosity >= 1:
self.stdout.write(" Installing indexes...\n")
# Install SQL indices for all newly created models
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(
model, no_style())
if index_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing index for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
with transaction.commit_on_success_unless_managed(
using=connection.alias):
for sql in index_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install index for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e))
finally:
cursor.close()
# Load initial_data fixtures (unless that has been disabled)
if self.load_initial_data:
for app_label in app_labels:
call_command('loaddata',
'initial_data',
verbosity=self.verbosity,
database=connection.alias,
skip_validation=True,
app_label=app_label,
hide_empty=True)
return created_models
def ready(self):
if DjangoTelegramBot.ready_run:
return
DjangoTelegramBot.ready_run = True
self.mode = WEBHOOK_MODE
if settings.DJANGO_TELEGRAMBOT.get('MODE', 'WEBHOOK') == 'POLLING':
self.mode = POLLING_MODE
modes = ['WEBHOOK', 'POLLING']
logger.info('Django Telegram Bot <{} mode>'.format(modes[self.mode]))
bots_list = settings.DJANGO_TELEGRAMBOT.get('BOTS', [])
if self.mode == WEBHOOK_MODE:
webhook_site = settings.DJANGO_TELEGRAMBOT.get(
'WEBHOOK_SITE', None)
if not webhook_site:
logger.warn(
'Required TELEGRAM_WEBHOOK_SITE missing in settings')
return
if webhook_site.endswith("/"):
webhook_site = webhook_site[:-1]
webhook_base = settings.DJANGO_TELEGRAMBOT.get(
'WEBHOOK_PREFIX', '/')
if webhook_base.startswith("/"):
webhook_base = webhook_base[1:]
if webhook_base.endswith("/"):
webhook_base = webhook_base[:-1]
cert = settings.DJANGO_TELEGRAMBOT.get('WEBHOOK_CERTIFICATE', None)
certificate = None
if cert and os.path.exists(cert):
logger.info('WEBHOOK_CERTIFICATE found in {}'.format(cert))
certificate = open(cert, 'rb')
elif cert:
logger.error(
'WEBHOOK_CERTIFICATE not found in {} '.format(cert))
for b in bots_list:
token = b.get('TOKEN', None)
context = b.get('CONTEXT', False)
if not token:
break
allowed_updates = b.get('ALLOWED_UPDATES', None)
timeout = b.get('TIMEOUT', None)
proxy = b.get('PROXY', None)
if self.mode == WEBHOOK_MODE:
try:
if b.get('MESSAGEQUEUE_ENABLED', False):
q = mq.MessageQueue(
all_burst_limit=b.get(
'MESSAGEQUEUE_ALL_BURST_LIMIT', 29),
all_time_limit_ms=b.get(
'MESSAGEQUEUE_ALL_TIME_LIMIT_MS', 1024))
if proxy:
request = Request(
proxy_url=proxy['proxy_url'],
urllib3_proxy_kwargs=proxy[
'urllib3_proxy_kwargs'],
con_pool_size=b.get(
'MESSAGEQUEUE_REQUEST_CON_POOL_SIZE', 8))
else:
request = Request(con_pool_size=b.get(
'MESSAGEQUEUE_REQUEST_CON_POOL_SIZE', 8))
bot = MQBot(token, request=request, mqueue=q)
else:
request = None
if proxy:
request = Request(proxy_url=proxy['proxy_url'],
urllib3_proxy_kwargs=proxy[
'urllib3_proxy_kwargs'])
bot = telegram.Bot(token=token, request=request)
DjangoTelegramBot.dispatchers.append(
Dispatcher(bot, None, workers=0, use_context=context))
hookurl = '{}/{}/{}/'.format(webhook_site, webhook_base,
token)
max_connections = b.get('WEBHOOK_MAX_CONNECTIONS', 40)
setted = bot.setWebhook(hookurl,
certificate=certificate,
timeout=timeout,
max_connections=max_connections,
allowed_updates=allowed_updates)
webhook_info = bot.getWebhookInfo()
real_allowed = webhook_info.allowed_updates if webhook_info.allowed_updates else [
"ALL"
]
bot.more_info = webhook_info
logger.info(
'Telegram Bot <{}> setting webhook [ {} ] max connections:{} allowed updates:{} pending updates:{} : {}'
.format(bot.username, webhook_info.url,
webhook_info.max_connections, real_allowed,
webhook_info.pending_update_count, setted))
except InvalidToken:
logger.error('Invalid Token : {}'.format(token))
return
except RetryAfter as er:
logger.debug(
'Error: "{}". Will retry in {} seconds'.format(
er.message, er.retry_after))
sleep(er.retry_after)
self.ready()
except TelegramError as er:
logger.error('Error: "{}"'.format(er.message))
return
else:
try:
updater = Updater(token=token,
request_kwargs=proxy,
use_context=context)
bot = updater.bot
bot.delete_webhook()
DjangoTelegramBot.updaters.append(updater)
DjangoTelegramBot.dispatchers.append(updater.dispatcher)
DjangoTelegramBot.__used_tokens.add(token)
except InvalidToken:
logger.error('Invalid Token : {}'.format(token))
return
except RetryAfter as er:
logger.debug(
'Error: "{}". Will retry in {} seconds'.format(
er.message, er.retry_after))
sleep(er.retry_after)
self.ready()
except TelegramError as er:
logger.error('Error: "{}"'.format(er.message))
return
DjangoTelegramBot.bots.append(bot)
DjangoTelegramBot.bot_tokens.append(token)
DjangoTelegramBot.bot_usernames.append(bot.username)
logger.debug('Telegram Bot <{}> set as default bot'.format(
DjangoTelegramBot.bots[0].username))
def module_imported(module_name, method_name, execute):
try:
m = importlib.import_module(module_name)
if execute and hasattr(m, method_name):
logger.debug('Run {}.{}()'.format(module_name,
method_name))
getattr(m, method_name)()
else:
logger.debug('Run {}'.format(module_name))
except ImportError as er:
if settings.DJANGO_TELEGRAMBOT.get('STRICT_INIT'):
raise er
else:
logger.error('{} : {}'.format(module_name, repr(er)))
return False
return True
# import telegram bot handlers for all INSTALLED_APPS
for app_config in apps.get_app_configs():
if module_has_submodule(app_config.module,
TELEGRAM_BOT_MODULE_NAME):
module_name = '%s.%s' % (app_config.name,
TELEGRAM_BOT_MODULE_NAME)
if module_imported(module_name, 'main', True):
logger.info('Loaded {}'.format(module_name))
num_bots = len(DjangoTelegramBot.__used_tokens)
if self.mode == POLLING_MODE and num_bots > 0:
logger.info(
'Please manually start polling update for {0} bot{1}. Run command{1}:'
.format(num_bots, 's' if num_bots > 1 else ''))
for token in DjangoTelegramBot.__used_tokens:
updater = DjangoTelegramBot.get_updater(bot_id=token)
logger.info('python manage.py botpolling --username={}'.format(
updater.bot.username))
def ready(self):
app.config_from_object('django.conf:settings', force=True)
install_apps = [
app_config.name for app_config in apps.get_app_configs()
]
app.autodiscover_tasks(lambda: install_apps, force=True)
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity'))
self.interactive = options.get('interactive')
self.show_traceback = options.get('traceback')
self.load_initial_data = options.get('load_initial_data')
self.test_database = options.get('test_database', False)
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_config in apps.get_app_configs():
if module_has_submodule(app_config.module, "management"):
import_module('.management', app_config.name)
# Get the database we're operating from
db = options.get('database')
connection = connections[db]
# If they asked for a migration listing, quit main execution flow and show it
if options.get("list", False):
return self.show_migration_list(connection, args)
# Work out which apps have migrations and which do not
executor = MigrationExecutor(connection,
self.migration_progress_callback)
# Before anything else, see if there's conflicting apps and drop out
# hard if there are any
conflicts = executor.loader.detect_conflicts()
if conflicts:
name_str = "; ".join("%s in %s" % (", ".join(names), app)
for app, names in conflicts.items())
raise CommandError(
"Conflicting migrations detected (%s).\nTo fix them run 'python manage.py makemigrations --merge'"
% name_str)
# If they supplied command line arguments, work out what they mean.
run_syncdb = False
target_app_labels_only = True
if len(args) > 2:
raise CommandError(
"Too many command-line arguments (expecting 'app_label' or 'app_label migrationname')"
)
elif len(args) == 2:
app_label, migration_name = args
if app_label not in executor.loader.migrated_apps:
raise CommandError(
"App '%s' does not have migrations (you cannot selectively sync unmigrated apps)"
% app_label)
if migration_name == "zero":
targets = [(app_label, None)]
else:
try:
migration = executor.loader.get_migration_by_prefix(
app_label, migration_name)
except AmbiguityError:
raise CommandError(
"More than one migration matches '%s' in app '%s'. Please be more specific."
% (migration_name, app_label))
except KeyError:
raise CommandError(
"Cannot find a migration matching '%s' from app '%s'."
% (migration_name, app_label))
targets = [(app_label, migration.name)]
target_app_labels_only = False
elif len(args) == 1:
app_label = args[0]
if app_label not in executor.loader.migrated_apps:
raise CommandError(
"App '%s' does not have migrations (you cannot selectively sync unmigrated apps)"
% app_label)
targets = [
key for key in executor.loader.graph.leaf_nodes()
if key[0] == app_label
]
else:
targets = executor.loader.graph.leaf_nodes()
run_syncdb = True
plan = executor.migration_plan(targets)
# Print some useful info
if self.verbosity >= 1:
self.stdout.write(
self.style.MIGRATE_HEADING("Operations to perform:"))
if run_syncdb and executor.loader.unmigrated_apps:
self.stdout.write(
self.style.MIGRATE_LABEL(" Synchronize unmigrated apps: ")
+ (", ".join(executor.loader.unmigrated_apps)))
if target_app_labels_only:
self.stdout.write(
self.style.MIGRATE_LABEL(" Apply all migrations: ") +
(", ".join(set(a for a, n in targets)) or "(none)"))
else:
if targets[0][1] is None:
self.stdout.write(
self.style.MIGRATE_LABEL(" Unapply all migrations: ")
+ "%s" % (targets[0][0], ))
else:
self.stdout.write(
self.style.MIGRATE_LABEL(
" Target specific migration: ") + "%s, from %s" %
(targets[0][1], targets[0][0]))
# Run the syncdb phase.
# If you ever manage to get rid of this, I owe you many, many drinks.
# Note that pre_migrate is called from inside here, as it needs
# the list of models about to be installed.
if run_syncdb and executor.loader.unmigrated_apps:
if self.verbosity >= 1:
self.stdout.write(
self.style.MIGRATE_HEADING(
"Synchronizing apps without migrations:"))
created_models = self.sync_apps(connection,
executor.loader.unmigrated_apps)
else:
created_models = []
emit_pre_migrate_signal([], self.verbosity, self.interactive,
connection.alias)
# The test runner requires us to flush after a syncdb but before migrations,
# so do that here.
if options.get("test_flush", False):
call_command(
'flush',
verbosity=max(self.verbosity - 1, 0),
interactive=False,
database=db,
reset_sequences=False,
inhibit_post_migrate=True,
)
# Migrate!
if self.verbosity >= 1:
self.stdout.write(
self.style.MIGRATE_HEADING("Running migrations:"))
if not plan:
if self.verbosity >= 1:
self.stdout.write(" No migrations to apply.")
# If there's changes that aren't in migrations yet, tell them how to fix it.
autodetector = MigrationAutodetector(
executor.loader.project_state(),
ProjectState.from_apps(apps),
)
changes = autodetector.changes(graph=executor.loader.graph)
if changes:
self.stdout.write(
self.style.NOTICE(
" Your models have changes that are not yet reflected in a migration, and so won't be applied."
))
self.stdout.write(
self.style.NOTICE(
" Run 'manage.py makemigrations' to make new migrations, and then re-run 'manage.py migrate' to apply them."
))
else:
executor.migrate(targets, plan, fake=options.get("fake", False))
# Send the post_migrate signal, so individual apps can do whatever they need
# to do at this point.
emit_post_migrate_signal(created_models, self.verbosity,
self.interactive, connection.alias)
def handle(self, *app_labels, **options):
format = options['format']
indent = options['indent']
using = options['database']
excludes = options['exclude']
output = options['output']
show_traceback = options['traceback']
use_natural_foreign_keys = options['use_natural_foreign_keys']
use_natural_primary_keys = options['use_natural_primary_keys']
use_base_manager = options['use_base_manager']
pks = options['primary_keys']
if pks:
primary_keys = pks.split(',')
else:
primary_keys = []
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
try:
model = apps.get_model(exclude)
except LookupError:
raise CommandError('Unknown model in excludes: %s' %
exclude)
excluded_models.add(model)
else:
try:
app_config = apps.get_app_config(exclude)
except LookupError as e:
raise CommandError(str(e))
excluded_apps.add(app_config)
if len(app_labels) == 0:
if primary_keys:
raise CommandError(
"You can only use --pks option with one model")
app_list = OrderedDict((app_config, None)
for app_config in apps.get_app_configs()
if app_config.models_module is not None
and app_config not in excluded_apps)
else:
if len(app_labels) > 1 and primary_keys:
raise CommandError(
"You can only use --pks option with one model")
app_list = OrderedDict()
for label in app_labels:
try:
app_label, model_label = label.split('.')
try:
app_config = apps.get_app_config(app_label)
except LookupError as e:
raise CommandError(str(e))
if app_config.models_module is None or app_config in excluded_apps:
continue
try:
model = app_config.get_model(model_label)
except LookupError:
raise CommandError("Unknown model: %s.%s" %
(app_label, model_label))
app_list_value = app_list.setdefault(app_config, [])
# We may have previously seen a "all-models" request for
# this app (no model qualifier was given). In this case
# there is no need adding specific models to the list.
if app_list_value is not None:
if model not in app_list_value:
app_list_value.append(model)
except ValueError:
if primary_keys:
raise CommandError(
"You can only use --pks option with one model")
# This is just an app - no model qualifier
app_label = label
try:
app_config = apps.get_app_config(app_label)
except LookupError as e:
raise CommandError(str(e))
if app_config.models_module is None or app_config in excluded_apps:
continue
app_list[app_config] = None
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
try:
serializers.get_serializer(format)
except serializers.SerializerDoesNotExist:
pass
raise CommandError("Unknown serialization format: %s" % format)
def get_objects(count_only=False):
"""
Collate the objects to be serialized. If count_only is True, just
count the number of objects to be serialized.
"""
models = serializers.sort_dependencies(app_list.items())
for model in models:
if model in excluded_models:
continue
if model._meta.proxy and model._meta.proxy_for_model not in models:
warnings.warn(
"%s is a proxy model and won't be serialized." %
model._meta.label,
category=ProxyModelWarning,
)
if not model._meta.proxy and router.allow_migrate_model(
using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(
model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
if count_only:
yield queryset.order_by().count()
else:
for obj in queryset.iterator():
yield obj
try:
self.stdout.ending = None
progress_output = None
object_count = 0
# If dumpdata is outputting to stdout, there is no way to display progress
if (output and self.stdout.isatty() and options['verbosity'] > 0):
progress_output = self.stdout
object_count = sum(get_objects(count_only=True))
stream = open(output, 'w') if output else None
try:
serializers.serialize(
format,
get_objects(),
indent=indent,
use_natural_foreign_keys=use_natural_foreign_keys,
use_natural_primary_keys=use_natural_primary_keys,
stream=stream or self.stdout,
progress_output=progress_output,
object_count=object_count)
finally:
if stream:
stream.close()
except Exception as e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
def handle_noargs(self, **options):
database = options.get('database')
connection = connections[database]
verbosity = options.get('verbosity')
interactive = options.get('interactive')
# The following are stealth options used by Django's internals.
reset_sequences = options.get('reset_sequences', True)
allow_cascade = options.get('allow_cascade', False)
inhibit_post_migrate = options.get('inhibit_post_migrate', False)
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_config in apps.get_app_configs():
try:
import_module('.management', app_config.name)
except ImportError:
pass
sql_list = sql_flush(self.style,
connection,
only_django=True,
reset_sequences=reset_sequences,
allow_cascade=allow_cascade)
if interactive:
confirm = input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to an empty state.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ %
connection.settings_dict['NAME'])
else:
confirm = 'yes'
if confirm == 'yes':
try:
with transaction.atomic(
using=database,
savepoint=connection.features.can_rollback_ddl):
with connection.cursor() as cursor:
for sql in sql_list:
cursor.execute(sql)
except Exception as e:
new_msg = (
"Database %s couldn't be flushed. Possible reasons:\n"
" * The database isn't running or isn't configured correctly.\n"
" * At least one of the expected database tables doesn't exist.\n"
" * The SQL was invalid.\n"
"Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.\n"
"The full error: %s") % (connection.settings_dict['NAME'],
e)
six.reraise(CommandError, CommandError(new_msg),
sys.exc_info()[2])
if not inhibit_post_migrate:
self.emit_post_migrate(verbosity, interactive, database)
# Reinstall the initial_data fixture.
if options.get('load_initial_data'):
# Reinstall the initial_data fixture.
call_command('loaddata', 'initial_data', **options)
else:
self.stdout.write("Flush cancelled.\n")
def sync_apps(self, connection, app_labels):
"Runs the old syncdb-style operation on a list of app_labels."
cursor = connection.cursor()
try:
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names(cursor)
created_models = set()
# Build the manifest of apps and models that are to be synchronized
all_models = [
(app_config.label,
router.get_migratable_models(app_config,
connection.alias,
include_auto_created=False))
for app_config in apps.get_app_configs()
if app_config.models_module is not None
and app_config.label in app_labels
]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(
using=connection.alias,
savepoint=connection.features.can_rollback_ddl):
deferred_sql = []
for app_name, model_list in manifest.items():
for model in model_list:
if not model._meta.can_migrate(connection):
continue
if self.verbosity >= 3:
self.stdout.write(
" Processing %s.%s model\n" %
(app_name, model._meta.object_name))
with connection.schema_editor() as editor:
if self.verbosity >= 1:
self.stdout.write(" Creating table %s\n" %
model._meta.db_table)
editor.create_model(model)
deferred_sql.extend(editor.deferred_sql)
editor.deferred_sql = []
created_models.add(model)
if self.verbosity >= 1:
self.stdout.write(" Running deferred SQL...\n")
for statement in deferred_sql:
cursor.execute(statement)
finally:
cursor.close()
return created_models
def creme_app_configs():
for app_config in apps.get_app_configs():
if app_config.creme_app:
yield app_config
