def serializer_factory(value):
if isinstance(value, Promise):
value = str(value)
elif isinstance(value, LazyObject):
# The unwrapped value is returned as the first item of the arguments
# tuple.
value = value.__reduce__()[1][0]
if isinstance(value, models.Field):
return ModelFieldSerializer(value)
if isinstance(value, models.manager.BaseManager):
return ModelManagerSerializer(value)
if isinstance(value, Operation):
return OperationSerializer(value)
if isinstance(value, type):
return TypeSerializer(value)
# Anything that knows how to deconstruct itself.
if hasattr(value, 'deconstruct'):
return DeconstructableSerializer(value)
for type_, serializer_cls in Serializer._registry.items():
if isinstance(value, type_):
return serializer_cls(value)
raise ValueError(
"Cannot serialize: %r\nThere are some values Django cannot serialize into "
"migration files.\nFor more, see https://docs.djangoproject.com/en/%s/"
"topics/migrations/#migration-serializing" % (value, get_docs_version())
)
def deconstruct(obj):
"""
Returns a 3-tuple of class import path, positional arguments,
and keyword arguments.
"""
# Python 2/fallback version
if path:
module_name, _, name = path.rpartition('.')
else:
module_name = obj.__module__
name = obj.__class__.__name__
# Make sure it's actually there and not an inner class
module = import_module(module_name)
if not hasattr(module, name):
raise ValueError(
"Could not find object %s in %s.\n"
"Please note that you cannot serialize things like inner "
"classes. Please move the object into the main module "
"body to use migrations.\n"
"For more information, see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#serializing-values"
% (name, module_name, get_docs_version()))
return (
path or '%s.%s' % (obj.__class__.__module__, name),
obj._constructor_args[0],
obj._constructor_args[1],
)
def serialize(self):
if getattr(self.value, "__self__", None) and isinstance(self.value.__self__, type):
klass = self.value.__self__
module = klass.__module__
return "%s.%s.%s" % (module, klass.__name__, self.value.__name__), {"import %s" % module}
# Further error checking
if self.value.__name__ == '<lambda>':
raise ValueError("Cannot serialize function: lambda")
if self.value.__module__ is None:
raise ValueError("Cannot serialize function %r: No module" % self.value)
# Python 3 is a lot easier, and only uses this branch if it's not local.
if getattr(self.value, "__qualname__", None) and getattr(self.value, "__module__", None):
if "<" not in self.value.__qualname__: # Qualname can include <locals>
return "%s.%s" % \
(self.value.__module__, self.value.__qualname__), {"import %s" % self.value.__module__}
# Python 2/fallback version
module_name = self.value.__module__
# Make sure it's actually there and not an unbound method
module = import_module(module_name)
if not hasattr(module, self.value.__name__):
raise ValueError(
"Could not find function %s in %s.\n"
"Please note that due to Python 2 limitations, you cannot "
"serialize unbound method functions (e.g. a method "
"declared and used in the same class body). Please move "
"the function into the main module body to use migrations.\n"
"For more information, see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#serializing-values"
% (self.value.__name__, module_name, get_docs_version())
)
# Needed on Python 2 only
if module_name == '__builtin__':
return self.value.__name__, set()
return "%s.%s" % (module_name, self.value.__name__), {"import %s" % module_name}
def csrf_failure(request, reason=""):
"""
Default view used when request fails CSRF protection
"""
from django.middleware.csrf import REASON_NO_REFERER, REASON_NO_CSRF_COOKIE
t = Engine().from_string(CSRF_FAILURE_TEMPLATE)
c = Context({
'title': _("Forbidden"),
'main': _("CSRF verification failed. Request aborted."),
'reason': reason,
'no_referer': reason == REASON_NO_REFERER,
'no_referer1': _(
"You are seeing this message because this HTTPS site requires a "
"'Referer header' to be sent by your Web browser, but none was "
"sent. This header is required for security reasons, to ensure "
"that your browser is not being hijacked by third parties."),
'no_referer2': _(
"If you have configured your browser to disable 'Referer' headers, "
"please re-enable them, at least for this site, or for HTTPS "
"connections, or for 'same-origin' requests."),
'no_cookie': reason == REASON_NO_CSRF_COOKIE,
'no_cookie1': _(
"You are seeing this message because this site requires a CSRF "
"cookie when submitting forms. This cookie is required for "
"security reasons, to ensure that your browser is not being "
"hijacked by third parties."),
'no_cookie2': _(
"If you have configured your browser to disable cookies, please "
"re-enable them, at least for this site, or for 'same-origin' "
"requests."),
'DEBUG': settings.DEBUG,
'docs_version': get_docs_version(),
'more': _("More information is available with DEBUG=True."),
})
return HttpResponseForbidden(t.render(c), content_type='text/html')
def render_multiple(self, model_states):
# We keep trying to render the models in a loop, ignoring invalid
# base errors, until the size of the unrendered models doesn't
# decrease by at least one, meaning there's a base dependency loop/
# missing base.
if not model_states:
return
# Prevent that all model caches are expired for each render.
with self.bulk_update():
unrendered_models = model_states
while unrendered_models:
new_unrendered_models = []
for model in unrendered_models:
try:
model.render(self)
except InvalidBasesError:
new_unrendered_models.append(model)
if len(new_unrendered_models) == len(unrendered_models):
raise InvalidBasesError(
"Cannot resolve bases for %r\nThis can happen if you are inheriting models from an "
"app with migrations (e.g. contrib.auth)\n in an app with no migrations; see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#dependencies "
"for more" % (new_unrendered_models, get_docs_version())
)
unrendered_models = new_unrendered_models
def render(self, include_real=None, ignore_swappable=False, skip_cache=False):
"Turns the project state into actual models in a new Apps"
if self.apps is None or skip_cache:
# Any apps in self.real_apps should have all their models included
# in the render. We don't use the original model instances as there
# are some variables that refer to the Apps object.
# FKs/M2Ms from real apps are also not included as they just
# mess things up with partial states (due to lack of dependencies)
real_models = []
for app_label in self.real_apps:
app = global_apps.get_app_config(app_label)
for model in app.get_models():
real_models.append(ModelState.from_model(model, exclude_rels=True))
# Populate the app registry with a stub for each application.
app_labels = set(model_state.app_label for model_state in self.models.values())
self.apps = Apps([AppConfigStub(label) for label in sorted(self.real_apps + list(app_labels))])
# We keep trying to render the models in a loop, ignoring invalid
# base errors, until the size of the unrendered models doesn't
# decrease by at least one, meaning there's a base dependency loop/
# missing base.
unrendered_models = list(self.models.values()) + real_models
while unrendered_models:
new_unrendered_models = []
for model in unrendered_models:
try:
model.render(self.apps)
except InvalidBasesError:
new_unrendered_models.append(model)
if len(new_unrendered_models) == len(unrendered_models):
raise InvalidBasesError(
"Cannot resolve bases for %r\nThis can happen if you are inheriting models from an "
"app with migrations (e.g. contrib.auth)\n in an app with no migrations; see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#dependencies "
"for more" % (new_unrendered_models, get_docs_version())
)
unrendered_models = new_unrendered_models
# make sure apps has no dangling references
if self.apps._pending_lookups:
# There's some lookups left. See if we can first resolve them
# ourselves - sometimes fields are added after class_prepared is sent
for lookup_model, operations in self.apps._pending_lookups.items():
try:
model = self.apps.get_model(lookup_model[0], lookup_model[1])
except LookupError:
app_label = "%s.%s" % (lookup_model[0], lookup_model[1])
if app_label == settings.AUTH_USER_MODEL and ignore_swappable:
continue
# Raise an error with a best-effort helpful message
# (only for the first issue). Error message should look like:
# "ValueError: Lookup failed for model referenced by
# field migrations.Book.author: migrations.Author"
msg = "Lookup failed for model referenced by field {field}: {model[0]}.{model[1]}"
raise ValueError(msg.format(field=operations[0][1], model=lookup_model))
else:
do_pending_lookups(model)
try:
return self.apps
finally:
if skip_cache:
self.apps = None
def csrf_failure(request, reason="", template_name=CSRF_FAILURE_TEMPLATE_NAME):
"""
Default view used when request fails CSRF protection
"""
from django.middleware.csrf import REASON_NO_CSRF_COOKIE, REASON_NO_REFERER
c = {
'title':
_("Forbidden"),
'main':
_("CSRF verification failed. Request aborted."),
'reason':
reason,
'no_referer':
reason == REASON_NO_REFERER,
'no_referer1':
_('You are seeing this message because this HTTPS site requires a '
'“Referer header” to be sent by your Web browser, but none was '
'sent. This header is required for security reasons, to ensure '
'that your browser is not being hijacked by third parties.'),
'no_referer2':
_('If you have configured your browser to disable “Referer” headers, '
'please re-enable them, at least for this site, or for HTTPS '
'connections, or for “same-origin” requests.'),
'no_referer3':
_('If you are using the <meta name="referrer" '
'content=\"no-referrer\"> tag or including the “Referrer-Policy: '
'no-referrer” header, please remove them. The CSRF protection '
'requires the “Referer” header to do strict referer checking. If '
'you’re concerned about privacy, use alternatives like '
'<a rel=\"noreferrer\" …> for links to third-party sites.'),
'no_cookie':
reason == REASON_NO_CSRF_COOKIE,
'no_cookie1':
_("You are seeing this message because this site requires a CSRF "
"cookie when submitting forms. This cookie is required for "
"security reasons, to ensure that your browser is not being "
"hijacked by third parties."),
'no_cookie2':
_('If you have configured your browser to disable cookies, please '
're-enable them, at least for this site, or for “same-origin” '
'requests.'),
'DEBUG':
settings.DEBUG,
'docs_version':
get_docs_version(),
'more':
_("More information is available with DEBUG=True."),
}
try:
t = loader.get_template(template_name)
except TemplateDoesNotExist:
if template_name == CSRF_FAILURE_TEMPLATE_NAME:
# If the default template doesn't exist, use the string template.
t = Engine().from_string(CSRF_FAILURE_TEMPLATE)
c = Context(c)
else:
# Raise if a developer-specified template doesn't exist.
raise
return HttpResponseForbidden(t.render(c), content_type='text/html')
def __init__(self, real_apps, models, ignore_swappable=False):
# Any apps in self.real_apps should have all their models included
# in the render. We don't use the original model instances as there
# are some variables that refer to the Apps object.
# FKs/M2Ms from real apps are also not included as they just
# mess things up with partial states (due to lack of dependencies)
real_models = []
for app_label in real_apps:
app = global_apps.get_app_config(app_label)
for model in app.get_models():
real_models.append(
ModelState.from_model(model, exclude_rels=True))
# Populate the app registry with a stub for each application.
app_labels = {model_state.app_label for model_state in models.values()}
app_configs = [
AppConfigStub(label)
for label in sorted(real_apps + list(app_labels))
]
super(StateApps, self).__init__(app_configs)
# We keep trying to render the models in a loop, ignoring invalid
# base errors, until the size of the unrendered models doesn't
# decrease by at least one, meaning there's a base dependency loop/
# missing base.
unrendered_models = list(models.values()) + real_models
while unrendered_models:
new_unrendered_models = []
for model in unrendered_models:
try:
model.render(self)
except InvalidBasesError:
new_unrendered_models.append(model)
if len(new_unrendered_models) == len(unrendered_models):
raise InvalidBasesError(
"Cannot resolve bases for %r\nThis can happen if you are inheriting models from an "
"app with migrations (e.g. contrib.auth)\n in an app with no migrations; see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#dependencies "
"for more" % (new_unrendered_models, get_docs_version()))
unrendered_models = new_unrendered_models
# If there are some lookups left, see if we can first resolve them
# ourselves - sometimes fields are added after class_prepared is sent
for lookup_model, operations in self._pending_lookups.items():
try:
model = self.get_model(lookup_model[0], lookup_model[1])
except LookupError:
app_label = "%s.%s" % (lookup_model[0], lookup_model[1])
if app_label == settings.AUTH_USER_MODEL and ignore_swappable:
continue
# Raise an error with a best-effort helpful message
# (only for the first issue). Error message should look like:
# "ValueError: Lookup failed for model referenced by
# field migrations.Book.author: migrations.Author"
msg = "Lookup failed for model referenced by field {field}: {model[0]}.{model[1]}"
raise ValueError(
msg.format(field=operations[0][1], model=lookup_model))
else:
do_pending_lookups(model)
def default_urlconf(request):
"""Create an empty URLconf 404 error response."""
with Path(CURRENT_DIR, 'templates', 'default_urlconf.html').open() as fh:
t = DEBUG_ENGINE.from_string(fh.read())
c = Context({
'version': get_docs_version(),
})
return HttpResponse(t.render(c), content_type='text/html')
def default_urlconf(request):
"""Create an empty URLconf 404 error response."""
with Path(CURRENT_DIR, 'templates', 'default_urlconf.html').open() as fh:
t = DEBUG_ENGINE.from_string(fh.read())
c = Context({
'version': get_docs_version(),
})
return HttpResponse(t.render(c), content_type='text/html')
def _generate_from_existing(
self,
project_id: str,
project_name: str,
project_dir: str,
cloud_sql_connection: str,
database_name: Optional[str] = None,
cloud_storage_bucket_name: Optional[str] = None):
"""Create Django settings file from an existing settings file.
We made several assumptions:
1. The existing settings file is in
<project_dir>/<project_name>/settings.py
2. There is only one settings file.
This is achieved by rename the existing settings file to
"base_settings.py", then create "local_settings.py" and
"cloud_settings.py" from our templates. "local_settings.py" and
"cloud_settings.py" inherits "base_settings.py", so the existing
settings file still have effects, and we only override what we need to.
Args:
project_id: GCP project id.
project_name: Name of the project to be created.
project_dir: The destination path to hold files of the project.
cloud_sql_connection: Connection string to allow the django app
to connect to the cloud sql proxy.
database_name: Name of your cloud database.
cloud_storage_bucket_name: Google Cloud Storage bucket name to
serve static content.
"""
database_name = database_name or project_name + '-db'
cloud_storage_bucket_name = cloud_storage_bucket_name or project_id
settings_templates_dir = os.path.join(
self._get_template_folder_path(),
self._SETTINGS_TEMPLATE_DIRECTORY)
django_dir = os.path.join(project_dir, project_name)
# TODO: Make it smart enough to find settings file instead of hard
# coding settings file path.
settings_file_path = os.path.join(django_dir, 'settings.py')
base_settings_path = os.path.join(django_dir, 'base_settings.py')
options = {
'project_id': project_id,
'project_name': project_name,
'docs_version': version.get_docs_version(),
'secret_key': utils.get_random_secret_key(),
'database_name': database_name,
'bucket_name': cloud_storage_bucket_name,
'cloud_sql_connection': cloud_sql_connection
}
self._render_directory(settings_templates_dir,
django_dir,
options=options)
os.replace(settings_file_path, base_settings_path)
def default_urlconf(request):
"""Create an empty URLconf 404 error response."""
with builtin_template_path('default_urlconf.html').open(
encoding='utf-8') as fh:
t = DEBUG_ENGINE.from_string(fh.read())
c = Context({
'version': get_docs_version(),
})
return HttpResponse(t.render(c), content_type='text/html')
def default_urlconf(request):
"""Create an empty URLconf 404 error response."""
with builtin_template_path("default_urlconf.html").open(
encoding="utf-8") as fh:
t = DEBUG_ENGINE.from_string(fh.read())
c = Context({
"version": get_docs_version(),
})
return HttpResponse(t.render(c))
def default_urlconf(request):
"""Create an empty URLconf 404 error response."""
with Path(CURRENT_DIR, "templates",
"default_urlconf.html").open(encoding="utf-8") as fh:
t = DEBUG_ENGINE.from_string(fh.read())
c = Context({
"version": get_docs_version(),
})
return HttpResponse(t.render(c), content_type="text/html")
def generate_from_existing(self,
project_id: str,
project_name: str,
cloud_sql_connection: str,
settings_path: str,
database_name: Optional[str] = None,
cloud_storage_bucket_name: Optional[str] = None,
file_storage_bucket_name: Optional[str] = None):
"""Create Django settings file from an existing settings file.
This is achieved by creating "cloud_settings.py" from our templates, and
make "cloud_settings.py" inherits the existing "settings.py", so the
existing settings file still have effects, and we only override what we
need to.
Args:
project_id: GCP project id.
project_name: Name of the project to be created.
cloud_sql_connection: Connection string to allow the django app
to connect to the cloud sql proxy.
settings_path: Absolute path of the settings.py used for deployment.
database_name: Name of your cloud database.
cloud_storage_bucket_name: Google Cloud Storage bucket name to
serve static content.
file_storage_bucket_name: Name of the Google Cloud Storage Bucket
used to store files by the Django app.
"""
database_name = database_name or project_name + '-db'
cloud_storage_bucket_name = cloud_storage_bucket_name or project_id
cloud_settings_template = os.path.join(
self._get_template_folder_path(),
self._SETTINGS_TEMPLATE_DIRECTORY, 'cloud_settings.py-tpl')
settings_dir = os.path.dirname(settings_path)
root, _ = os.path.splitext(settings_path)
module_relative_path = os.path.relpath(root, settings_dir)
settings_module = module_relative_path.replace('/', '.')
options = {
'project_id': project_id,
'project_name': project_name,
'docs_version': version.get_docs_version(),
'secret_key': django_utils.get_random_secret_key(),
'settings_module': settings_module,
'database_name': database_name,
'bucket_name': cloud_storage_bucket_name,
'file_bucket_name': file_storage_bucket_name,
'cloud_sql_connection': cloud_sql_connection
}
settings_output_path = os.path.join(settings_dir, 'cloud_settings.py')
self._render_file(cloud_settings_template,
settings_output_path,
options=options)
def _check_sql_mode(self, **kwargs):
if not (self.connection.sql_mode & {'STRICT_TRANS_TABLES', 'STRICT_ALL_TABLES'}):
return [checks.Warning(
"MySQL Strict Mode is not set for database connection '%s'" % self.connection.alias,
hint="MySQL's Strict Mode fixes many data integrity problems in MySQL, "
"such as data truncation upon insertion, by escalating warnings into "
"errors. It is strongly recommended you activate it. See: "
"https://docs.djangoproject.com/en/%s/ref/databases/#mysql-sql-mode"
% (get_docs_version(),),
id='mysql.W002',
)]
return []
def csrf_failure(request, reason="", template_name=CSRF_FAILURE_TEMPLATE_NAME):
"""
Default view used when request fails CSRF protection
"""
from django.middleware.csrf import REASON_NO_REFERER, REASON_NO_CSRF_COOKIE
c = {
'title':
_("Forbidden"),
'main':
_("CSRF verification failed. Request aborted."),
'reason':
reason,
'no_referer':
reason == REASON_NO_REFERER,
'no_referer1':
_("You are seeing this message because this HTTPS site requires a "
"'Referer header' to be sent by your Web browser, but none was "
"sent. This header is required for security reasons, to ensure "
"that your browser is not being hijacked by third parties."),
'no_referer2':
_("If you have configured your browser to disable 'Referer' headers, "
"please re-enable them, at least for this site, or for HTTPS "
"connections, or for 'same-origin' requests."),
'no_cookie':
reason == REASON_NO_CSRF_COOKIE,
'no_cookie1':
_("You are seeing this message because this site requires a CSRF "
"cookie when submitting forms. This cookie is required for "
"security reasons, to ensure that your browser is not being "
"hijacked by third parties."),
'no_cookie2':
_("If you have configured your browser to disable cookies, please "
"re-enable them, at least for this site, or for 'same-origin' "
"requests."),
'DEBUG':
settings.DEBUG,
'docs_version':
get_docs_version(),
'more':
_("More information is available with DEBUG=True."),
}
try:
t = loader.get_template(template_name)
except TemplateDoesNotExist:
if template_name == CSRF_FAILURE_TEMPLATE_NAME:
# If the default template doesn't exist, use the string template.
t = Engine().from_string(CSRF_FAILURE_TEMPLATE)
c = Context(c)
else:
# Raise if a developer-specified template doesn't exist.
raise
return HttpResponseForbidden(t.render(c), content_type='text/html')
def test_invalid_path(self):
obj = DeconstructibleInvalidPathClass()
docs_version = get_docs_version()
msg = (f'Could not find object DeconstructibleInvalidPathClass in '
f'utils_tests.deconstructible_classes.\n'
f'Please note that you cannot serialize things like inner '
f'classes. Please move the object into the main module body to '
f'use migrations.\n'
f'For more information, see '
f'https://docs.djangoproject.com/en/{docs_version}/topics/'
f'migrations/#serializing-values')
with self.assertRaisesMessage(ValueError, msg):
obj.deconstruct()
def csrf_failure(request, reason="", template_name=CSRF_FAILURE_TEMPLATE_NAME):
"""
Default view used when request fails CSRF protection
"""
from django.middleware.csrf import REASON_NO_REFERER, REASON_NO_CSRF_COOKIE
c = {
'title': _("Forbidden"),
'main': _("CSRF verification failed. Request aborted."),
'reason': reason,
'no_referer': reason == REASON_NO_REFERER,
'no_referer1': _(
"You are seeing this message because this HTTPS site requires a "
"'Referer header' to be sent by your Web browser, but none was "
"sent. This header is required for security reasons, to ensure "
"that your browser is not being hijacked by third parties."),
'no_referer2': _(
"If you have configured your browser to disable 'Referer' headers, "
"please re-enable them, at least for this site, or for HTTPS "
"connections, or for 'same-origin' requests."),
'no_referer3': _(
"If you are using the <meta name=\"referrer\" "
"content=\"no-referrer\"> tag or including the 'Referrer-Policy: "
"no-referrer' header, please remove them. The CSRF protection "
"requires the 'Referer' header to do strict referer checking. If "
"you're concerned about privacy, use alternatives like "
"<a rel=\"noreferrer\" ...> for links to third-party sites."),
'no_cookie': reason == REASON_NO_CSRF_COOKIE,
'no_cookie1': _(
"You are seeing this message because this site requires a CSRF "
"cookie when submitting forms. This cookie is required for "
"security reasons, to ensure that your browser is not being "
"hijacked by third parties."),
'no_cookie2': _(
"If you have configured your browser to disable cookies, please "
"re-enable them, at least for this site, or for 'same-origin' "
"requests."),
'DEBUG': settings.DEBUG,
'docs_version': get_docs_version(),
'more': _("More information is available with DEBUG=True."),
}
try:
t = loader.get_template(template_name)
except TemplateDoesNotExist:
if template_name == CSRF_FAILURE_TEMPLATE_NAME:
# If the default template doesn't exist, use the string template.
t = Engine().from_string(CSRF_FAILURE_TEMPLATE)
c = Context(c)
else:
# Raise if a developer-specified template doesn't exist.
raise
return HttpResponseForbidden(t.render(c), content_type='text/html')
def _prepare_context(self, options):
"""
Создание контекста
"""
self.context = Context(
{
**options,
self.base_name: self.name,
self.base_directory: self._top_dir_path,
self.camel_case_name: self.camel_case_value,
self.base_python_path: self._python_path_value,
'docs_version': get_docs_version(),
'django_version': django.__version__,
},
autoescape=False)
def sql_create(app_config, style, connection):
"Returns a list of the CREATE TABLE SQL statements for the given app."
check_for_migrations(app_config, connection)
if connection.settings_dict['ENGINE'] == 'django.db.backends.dummy':
# This must be the "dummy" database backend, which means the user
# hasn't set ENGINE for the database.
raise CommandError(
"Django doesn't know which syntax to use for your SQL statements,\n"
"because you haven't properly specified the ENGINE setting for the database.\n"
"see: https://docs.djangoproject.com/en/%s/ref/settings/#databases" % get_docs_version()
)
# Get installed models, so we generate REFERENCES right.
# We trim models from the current app so that the sqlreset command does not
# generate invalid SQL (leaving models out of known_models is harmless, so
# we can be conservative).
app_models = list(app_config.get_models(include_auto_created=True))
final_output = []
tables = connection.introspection.table_names()
known_models = set(model for model in connection.introspection.installed_models(tables) if model not in app_models)
pending_references = {}
for model in router.get_migratable_models(app_config, connection.alias, include_auto_created=True):
output, references = connection.creation.sql_create_model(model, style, known_models)
final_output.extend(output)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in known_models:
final_output.extend(connection.creation.sql_for_pending_references(refto, style, pending_references))
final_output.extend(connection.creation.sql_for_pending_references(model, style, pending_references))
# Keep track of the fact that we've created the table for this model.
known_models.add(model)
# Handle references to tables that are from other apps
# but don't exist physically.
not_installed_models = set(pending_references.keys())
if not_installed_models:
alter_sql = []
for model in not_installed_models:
alter_sql.extend('-- ' + sql for sql in
connection.creation.sql_for_pending_references(model, style, pending_references))
if alter_sql:
final_output.append('-- The following references should be added but depend on non-existent tables:')
final_output.extend(alter_sql)
return final_output
def _check_sql_mode(self, **kwargs):
with self.connection.cursor() as cursor:
cursor.execute("SELECT @@sql_mode")
sql_mode = cursor.fetchone()
modes = set(sql_mode[0].split(',') if sql_mode else ())
if not (modes & {'STRICT_TRANS_TABLES', 'STRICT_ALL_TABLES'}):
return [checks.Warning(
"MySQL Strict Mode is not set for database connection '%s'" % self.connection.alias,
hint="MySQL's Strict Mode fixes many table_s integrity problems in MySQL, "
"such as table_s truncation upon insertion, by escalating warnings into "
"errors. It is strongly recommended you activate it. See: "
"https://docs.djangoproject.com/en/%s/ref/databases/#mysql-sql-mode"
% (get_docs_version(),),
id='mysql.W002',
)]
return []
def _check_sql_mode(self, **kwargs):
with self.connection.cursor() as cursor:
cursor.execute("SELECT @@sql_mode")
sql_mode = cursor.fetchone()
modes = set(sql_mode[0].split(','))
if not (modes & {'STRICT_TRANS_TABLES', 'STRICT_ALL_TABLES'}):
return [checks.Warning(
"MySQL Strict Mode is not set for database connection '%s'" % self.connection.alias,
hint="MySQL's Strict Mode fixes many data integrity problems in MySQL, "
"such as data truncation upon insertion, by escalating warnings into "
"errors. It is strongly recommended you activate it. See: "
"https://docs.djangoproject.com/en/%s/ref/databases/#mysql-sql-mode"
% (get_docs_version(),),
id='mysql.W002',
)]
return []
def generate_new(self, project_name: str, project_dir: str, app_name: str):
"""Generate django project files using our template.
Args:
project_name: Name of the project to be created.
project_dir: The destination path to hold files of the project.
app_name: The app that you want to create in your project.
"""
options = {
'app_name': app_name,
'project_name': project_name,
'docs_version': version.get_docs_version(),
}
filename_template_replacement = {
'project_name': project_name,
}
self._generate_files(self.PROJECT_TEMPLATE_FOLDER, project_dir,
filename_template_replacement, options)
def csrf_failure(request, reason=""):
from django.middleware.csrf import REASON_BAD_TOKEN, REASON_NO_REFERER, REASON_NO_CSRF_COOKIE
t = loader.get_template("403_csrf.html")
c = Context({
'title':
_("Forbidden"),
'main':
_("CSRF verification failed. Request aborted."),
'reason':
reason,
'bad_token':
reason == REASON_BAD_TOKEN,
'no_referer':
reason == REASON_NO_REFERER,
'no_referer1':
_("You are seeing this message because this HTTPS site requires a "
"'Referer header' to be sent by your Web browser, but none was "
"sent. This header is required for security reasons, to ensure "
"that your browser is not being hijacked by third parties."),
'no_referer2':
_("If you have configured your browser to disable 'Referer' headers, "
"please re-enable them, at least for this site, or for HTTPS "
"connections, or for 'same-origin' requests."),
'no_cookie':
reason == REASON_NO_CSRF_COOKIE,
'no_cookie1':
_("You are seeing this message because this site requires a CSRF "
"cookie when submitting forms. This cookie is required for "
"security reasons, to ensure that your browser is not being "
"hijacked by third parties."),
'no_cookie2':
_("If you have configured your browser to disable cookies, please "
"re-enable them, at least for this site, or for 'same-origin' "
"requests."),
'DEBUG':
settings.DEBUG,
'docs_version':
get_docs_version(),
'more':
_("More information is available with DEBUG=True."),
})
c = RequestContext(request, c)
return HttpResponseForbidden(t.render(c), content_type='text/html')
def generate_new(self,
project_id: str,
project_name: str,
project_dir: str,
cloud_sql_connection: str,
database_name: Optional[str] = None,
cloud_storage_bucket_name: Optional[str] = None,
file_storage_bucket_name: Optional[str] = None):
"""Create Django settings file using our template.
Args:
project_id: GCP project id.
project_name: Name of the project to be created.
project_dir: The destination path to hold files of the project.
cloud_sql_connection: Connection string to allow the django app
to connect to the cloud sql proxy.
database_name: Name of your cloud database.
cloud_storage_bucket_name: Google Cloud Storage bucket name to
serve static content.
file_storage_bucket_name: Name of the Google Cloud Storage Bucket
used to store files by the Django app.
"""
database_name = database_name or project_name + '-db'
destination = os.path.join(
os.path.abspath(os.path.expanduser(project_dir)), project_name)
cloud_storage_bucket_name = cloud_storage_bucket_name or project_id
settings_templates_dir = os.path.join(
self._get_template_folder_path(),
self._SETTINGS_TEMPLATE_DIRECTORY)
options = {
'project_id': project_id,
'project_name': project_name,
'docs_version': version.get_docs_version(),
'secret_key': django_utils.get_random_secret_key(),
'settings_module': 'settings',
'database_name': database_name,
'bucket_name': cloud_storage_bucket_name,
'file_bucket_name': file_storage_bucket_name,
'cloud_sql_connection': cloud_sql_connection
}
self._render_directory(settings_templates_dir,
destination,
options=options)
def __init__(self, func, name=None):
if PY36:
self.real_func = func
else:
func_name = func.__name__
name = name or func_name
if not (isinstance(name, str) and name.isidentifier()):
raise ValueError(
"%r can't be used as the name of a cached_property." %
name, )
if self._is_mangled(name):
raise ValueError(
'cached_property does not work with mangled methods on '
'Python < 3.6 without the appropriate `name` argument. See '
'https://docs.djangoproject.com/en/%s/ref/utils/'
'#cached-property-mangled-name' % get_docs_version(), )
self.name = name
self.func = func
self.__doc__ = getattr(func, '__doc__')
def test_too_long_char_field_under_mysql(self):
from django.db.backends.mysql.validation import DatabaseValidation
class Model(models.Model):
field = models.CharField(unique=True, max_length=256)
field = Model._meta.get_field('field')
validator = DatabaseValidation(connection=connection)
self.assertEqual(validator.check_field(field), [
DjangoWarning(
'%s may not allow unique CharFields to have a max_length > '
'255.' % connection.display_name,
hint=(
'See: https://docs.djangoproject.com/en/%s/ref/databases/'
'#mysql-character-fields' % get_docs_version()),
obj=field,
id='mysql.W003',
)
])
def csrf_failure(request, reason="", template_name=CSRF_FAILURE_TEMPLATE_NAME):
"""
Default view used when request fails CSRF protection
"""
from django.middleware.csrf import REASON_NO_REFERER, REASON_NO_CSRF_COOKIE
c = {
'title': _("Forbidden"),
'main': _("CSRF verification failed. Request aborted."),
'reason': reason,
'no_referer': reason == REASON_NO_REFERER,
'no_referer1': _(
"You are seeing this message because this HTTPS site requires a "
"'Referer header' to be sent by your Web browser, but none was "
"sent. This header is required for security reasons, to ensure "
"that your browser is not being hijacked by third parties."),
'no_referer2': _(
"If you have configured your browser to disable 'Referer' headers, "
"please re-enable them, at least for this site, or for HTTPS "
"connections, or for 'same-origin' requests."),
<<<<<<< HEAD
=======
'no_referer3': _(
"If you are using the <meta name=\"referrer\" "
"content=\"no-referrer\"> tag or including the 'Referrer-Policy: "
"no-referrer' header, please remove them. The CSRF protection "
"requires the 'Referer' header to do strict referer checking. If "
"you're concerned about privacy, use alternatives like "
"<a rel=\"noreferrer\" ...> for links to third-party sites."),
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
'no_cookie': reason == REASON_NO_CSRF_COOKIE,
'no_cookie1': _(
"You are seeing this message because this site requires a CSRF "
"cookie when submitting forms. This cookie is required for "
"security reasons, to ensure that your browser is not being "
"hijacked by third parties."),
'no_cookie2': _(
"If you have configured your browser to disable cookies, please "
"re-enable them, at least for this site, or for 'same-origin' "
"requests."),
'DEBUG': settings.DEBUG,
'docs_version': get_docs_version(),
'more': _("More information is available with DEBUG=True."),
}
def test_sqlall(capsys, mocker, settings):
mocker.patch(
'django.db.backends.base.schema.BaseDatabaseSchemaEditor'
'._create_index_name',
return_value='INDEX_NAME')
call_command('sqlall', 'north_app')
captured = capsys.readouterr()
assert captured.out == (
'BEGIN;\n'
'CREATE TABLE "north_app_author" '
'("id" serial NOT NULL PRIMARY KEY, "name" varchar(100) NOT NULL)\n'
'CREATE TABLE "north_app_reader" '
'("id" serial NOT NULL PRIMARY KEY, "name" varchar(100) NOT NULL)\n'
'CREATE TABLE "north_app_book_readers" '
'("id" serial NOT NULL PRIMARY KEY, '
'"book_id" integer NOT NULL, '
'"reader_id" integer NOT NULL)\n'
'CREATE TABLE "north_app_book" '
'("id" serial NOT NULL PRIMARY KEY, '
'"author_id" integer NOT NULL, '
'"title" varchar(100) NOT NULL, '
'"pages" integer NOT NULL)\n'
'ALTER TABLE "north_app_book_readers" '
'ADD CONSTRAINT "INDEX_NAME" FOREIGN KEY ("book_id") '
'REFERENCES "north_app_book" ("id") DEFERRABLE INITIALLY DEFERRED\n'
'ALTER TABLE "north_app_book_readers" '
'ADD CONSTRAINT "INDEX_NAME" FOREIGN KEY ("reader_id") '
'REFERENCES "north_app_reader" ("id") DEFERRABLE INITIALLY DEFERRED\n'
'ALTER TABLE "north_app_book_readers" ' +
('ADD CONSTRAINT INDEX_NAME UNIQUE ("book_id", "reader_id")\n'
if get_docs_version() == '2.0' else
'ADD CONSTRAINT "INDEX_NAME" UNIQUE ("book_id", "reader_id")\n') +
'CREATE INDEX "INDEX_NAME" ON "north_app_book_readers" ("book_id")\n'
'CREATE INDEX "INDEX_NAME" ON "north_app_book_readers" ("reader_id")\n'
'ALTER TABLE "north_app_book" '
'ADD CONSTRAINT "INDEX_NAME" '
'FOREIGN KEY ("author_id") REFERENCES "north_app_author" ("id") '
'DEFERRABLE INITIALLY DEFERRED\n'
'CREATE INDEX "INDEX_NAME" ON "north_app_book" ("author_id")\n' +
'COMMIT;\n')
def __init__(self, func, name=None):
if PY36:
self.real_func = func
else:
func_name = func.__name__
name = name or func_name
if not (isinstance(name, str) and name.isidentifier()):
raise ValueError(
"%r can't be used as the name of a cached_property." % name,
)
if self._is_mangled(name):
raise ValueError(
'cached_property does not work with mangled methods on '
'Python < 3.6 without the appropriate `name` argument. See '
'https://docs.djangoproject.com/en/%s/ref/utils/'
'#cached-property-mangled-name' % get_docs_version(),
)
self.name = name
self.func = func
self.__doc__ = getattr(func, '__doc__')
def ask_unique_callable_default_addition(self, field_name, model_name):
"""Adding a unique field with a callable default."""
if not self.dry_run:
version = get_docs_version()
choice = self._choice_input(
f'Callable default on unique field {model_name}.{field_name} '
f'will not generate unique values upon migrating.\n'
f'Please choose how to proceed:\n',
[
f'Continue making this migration as the first step in '
f'writing a manual migration to generate unique values '
f'described here: '
f'https://docs.djangoproject.com/en/{version}/howto/'
f'writing-migrations/#migrations-that-add-unique-fields.',
'Quit and edit field options in models.py.',
],
)
if choice == 2:
sys.exit(3)
return None
def render_multiple(self, model_states):
# We keep trying to render the models in a loop, ignoring invalid
# base errors, until the size of the unrendered models doesn't
# decrease by at least one, meaning there's a base dependency loop/
# missing base.
unrendered_models = model_states
while unrendered_models:
new_unrendered_models = []
for model in unrendered_models:
try:
model.render(self)
except InvalidBasesError:
new_unrendered_models.append(model)
if len(new_unrendered_models) == len(unrendered_models):
raise InvalidBasesError(
"Cannot resolve bases for %r\nThis can happen if you are inheriting models from an "
"app with migrations (e.g. contrib.auth)\n in an app with no migrations; see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#dependencies "
"for more" % (new_unrendered_models, get_docs_version()))
unrendered_models = new_unrendered_models
def test_sqlall(mocker, settings):
stdout = mocker.patch('sys.stdout', new_callable=StringIO)
mocker.patch(
'django.db.backends.base.schema.BaseDatabaseSchemaEditor'
'._create_index_name',
return_value='INDEX_NAME')
call_command('sqlall', 'north_app')
assert stdout.getvalue() == (
'BEGIN;\n'
'CREATE TABLE "north_app_author" '
'("id" serial NOT NULL PRIMARY KEY, "name" varchar(100) NOT NULL)\n'
'CREATE TABLE "north_app_reader" '
'("id" serial NOT NULL PRIMARY KEY, "name" varchar(100) NOT NULL)\n'
'CREATE TABLE "north_app_book_readers" '
'("id" serial NOT NULL PRIMARY KEY, '
'"book_id" integer NOT NULL, '
'"reader_id" integer NOT NULL)\n'
'CREATE TABLE "north_app_book" '
'("id" serial NOT NULL PRIMARY KEY, '
'"author_id" integer NOT NULL, '
'"title" varchar(100) NOT NULL, '
'"pages" integer NOT NULL)\n'
'ALTER TABLE "north_app_book_readers" '
'ADD CONSTRAINT "INDEX_NAME" FOREIGN KEY ("book_id") '
'REFERENCES "north_app_book" ("id") DEFERRABLE INITIALLY DEFERRED\n'
'ALTER TABLE "north_app_book_readers" '
'ADD CONSTRAINT "INDEX_NAME" FOREIGN KEY ("reader_id") '
'REFERENCES "north_app_reader" ("id") DEFERRABLE INITIALLY DEFERRED\n'
'ALTER TABLE "north_app_book_readers" '
'ADD CONSTRAINT "INDEX_NAME" UNIQUE ("book_id", "reader_id")\n'
'CREATE INDEX "INDEX_NAME" ON "north_app_book_readers" ("book_id")\n'
'CREATE INDEX "INDEX_NAME" ON "north_app_book_readers" ("reader_id")\n'
'ALTER TABLE "north_app_book" '
'ADD CONSTRAINT "INDEX_NAME" '
'FOREIGN KEY ("author_id") REFERENCES "north_app_author" ("id") '
'DEFERRABLE INITIALLY DEFERRED\n'
'CREATE INDEX "INDEX_NAME" ON "north_app_book" ("author_id")\n' +
'\n' * (0 if get_docs_version() in ['1.10', '1.11'] else 1) +
'COMMIT;\n')
def _check_sql_mode(self, **kwargs):
if not (self.connection.sql_mode
& {"STRICT_TRANS_TABLES", "STRICT_ALL_TABLES"}):
return [
checks.Warning(
"%s Strict Mode is not set for database connection '%s'" %
(self.connection.display_name, self.connection.alias),
hint=
("%s's Strict Mode fixes many data integrity problems in "
"%s, such as data truncation upon insertion, by "
"escalating warnings into errors. It is strongly "
"recommended you activate it. See: "
"https://docs.djangoproject.com/en/%s/ref/databases/"
"#mysql-sql-mode" % (
self.connection.display_name,
self.connection.display_name,
get_docs_version(),
), ),
id="mysql.W002",
)
]
return []
def serialize(self):
if getattr(self.value, "__self__", None) and isinstance(
self.value.__self__, type):
klass = self.value.__self__
module = klass.__module__
return "%s.%s.%s" % (module, klass.__name__,
self.value.__name__), {"import %s" % module}
# Further error checking
if self.value.__name__ == '<lambda>':
raise ValueError("Cannot serialize function: lambda")
if self.value.__module__ is None:
raise ValueError("Cannot serialize function %r: No module" %
self.value)
# Python 3 is a lot easier, and only uses this branch if it's not local.
if getattr(self.value, "__qualname__", None) and getattr(
self.value, "__module__", None):
if "<" not in self.value.__qualname__: # Qualname can include <locals>
return "%s.%s" % \
(self.value.__module__, self.value.__qualname__), {"import %s" % self.value.__module__}
# Python 2/fallback version
module_name = self.value.__module__
# Make sure it's actually there and not an unbound method
module = import_module(module_name)
if not hasattr(module, self.value.__name__):
raise ValueError(
"Could not find function %s in %s.\n"
"Please note that due to Python 2 limitations, you cannot "
"serialize unbound method functions (e.g. a method "
"declared and used in the same class body). Please move "
"the function into the main module body to use migrations.\n"
"For more information, see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#serializing-values"
% (self.value.__name__, module_name, get_docs_version()))
# Needed on Python 2 only
if module_name == '__builtin__':
return self.value.__name__, set()
return "%s.%s" % (module_name, self.value.__name__), {
"import %s" % module_name
}
def handle(self, app_or_project, name, target=None, **options):
self.app_or_project = app_or_project
self.a_or_an = 'an' if app_or_project == 'app' else 'a'
self.paths_to_remove = []
self.verbosity = options['verbosity']
self.validate_name(name)
# if some directory is given, make sure it's nicely expanded
if target is None:
top_dir = os.path.join(os.getcwd(), name)
try:
os.makedirs(top_dir)
except FileExistsError:
raise CommandError("'%s' already exists" % top_dir)
except OSError as e:
raise CommandError(e)
else:
if app_or_project == 'app':
self.validate_name(os.path.basename(target), 'directory')
top_dir = os.path.abspath(os.path.expanduser(target))
if not os.path.exists(top_dir):
raise CommandError("Destination directory '%s' does not "
"exist, please create it first." % top_dir)
extensions = tuple(handle_extensions(options['extensions']))
extra_files = []
for file in options['files']:
extra_files.extend(map(lambda x: x.strip(), file.split(',')))
if self.verbosity >= 2:
self.stdout.write(
'Rendering %s template files with extensions: %s' %
(app_or_project, ', '.join(extensions)))
self.stdout.write(
'Rendering %s template files with filenames: %s' %
(app_or_project, ', '.join(extra_files)))
base_name = '%s_name' % app_or_project
base_subdir = '%s_template' % app_or_project
base_directory = '%s_directory' % app_or_project
camel_case_name = 'camel_case_%s_name' % app_or_project
camel_case_value = ''.join(x for x in name.title() if x != '_')
context = Context(
{
**options,
base_name: name,
base_directory: top_dir,
camel_case_name: camel_case_value,
'docs_version': get_docs_version(),
'django_version': django.__version__,
},
autoescape=False)
# Setup a stub settings environment for template rendering
if not settings.configured:
settings.configure()
django.setup()
template_dir = self.handle_template(options['template'], base_subdir)
prefix_length = len(template_dir) + 1
for root, dirs, files in os.walk(template_dir):
path_rest = root[prefix_length:]
relative_dir = path_rest.replace(base_name, name)
if relative_dir:
target_dir = os.path.join(top_dir, relative_dir)
os.makedirs(target_dir, exist_ok=True)
for dirname in dirs[:]:
if dirname.startswith('.') or dirname == '__pycache__':
dirs.remove(dirname)
for filename in files:
if filename.endswith(('.pyo', '.pyc', '.py.class')):
# Ignore some files as they cause various breakages.
continue
old_path = os.path.join(root, filename)
new_path = os.path.join(top_dir, relative_dir,
filename.replace(base_name, name))
for old_suffix, new_suffix in self.rewrite_template_suffixes:
if new_path.endswith(old_suffix):
new_path = new_path[:-len(old_suffix)] + new_suffix
break # Only rewrite once
if os.path.exists(new_path):
raise CommandError(
"%s already exists. Overlaying %s %s into an existing "
"directory won't replace conflicting files." % (
new_path,
self.a_or_an,
app_or_project,
))
# Only render the Python files, as we don't want to
# accidentally render Django templates files
if new_path.endswith(extensions) or filename in extra_files:
with open(old_path, encoding='utf-8') as template_file:
content = template_file.read()
template = Engine().from_string(content)
content = template.render(context)
with open(new_path, 'w', encoding='utf-8') as new_file:
new_file.write(content)
else:
shutil.copyfile(old_path, new_path)
if self.verbosity >= 2:
self.stdout.write('Creating %s' % new_path)
try:
shutil.copymode(old_path, new_path)
self.make_writeable(new_path)
except OSError:
self.stderr.write(
"Notice: Couldn't set permission bits on %s. You're "
"probably using an uncommon filesystem setup. No "
"problem." % new_path, self.style.NOTICE)
if self.paths_to_remove:
if self.verbosity >= 2:
self.stdout.write('Cleaning up temporary files.')
for path_to_remove in self.paths_to_remove:
if os.path.isfile(path_to_remove):
os.remove(path_to_remove)
else:
shutil.rmtree(path_to_remove)
from __future__ import unicode_literals
class TemplateCommand(BaseCommand):
"""
Copy either a Django application layout template or a Django project
layout template into the specified directory.
:param style: A color style object (see django.core.management.color).
:param app_or_project: The string 'app' or 'project'.
:param name: The name of the application or project.
:param directory: The directory to which the template should be copied.
:param options: The additional variables passed to project or app templates
"""
requires_system_checks = []
# The supported URL schemes
url_schemes = ["http", "https", "ftp"]
# Rewrite the following suffixes when determining the target filename.
rewrite_template_suffixes = (
# Allow shipping invalid .py files without byte-compilation.
(".py-tpl", ".py"), )
def add_arguments(self, parser):
parser.add_argument("name", help="Name of the application or project.")
parser.add_argument("directory",
nargs="?",
help="Optional destination directory")
parser.add_argument("--template",
help="The path or URL to load the template from.")
parser.add_argument(
"--extension",
"-e",
dest="extensions",
action="append",
default=["py"],
help='The file extension(s) to render (default: "py"). '
"Separate multiple extensions with commas, or use "
"-e multiple times.",
)
parser.add_argument(
"--name",
"-n",
dest="files",
action="append",
default=[],
help="The file name(s) to render. Separate multiple file names "
"with commas, or use -n multiple times.",
)
parser.add_argument(
"--exclude",
"-x",
action="append",
default=argparse.SUPPRESS,
nargs="?",
const="",
help=("The directory name(s) to exclude, in addition to .git and "
"__pycache__. Can be used multiple times."),
)
def handle(self, app_or_project, name, target=None, **options):
self.app_or_project = app_or_project
self.a_or_an = "an" if app_or_project == "app" else "a"
self.paths_to_remove = []
self.verbosity = options["verbosity"]
self.validate_name(name)
# if some directory is given, make sure it's nicely expanded
if target is None:
top_dir = os.path.join(os.getcwd(), name)
try:
os.makedirs(top_dir)
except FileExistsError:
raise CommandError("'%s' already exists" % top_dir)
except OSError as e:
raise CommandError(e)
else:
top_dir = os.path.abspath(os.path.expanduser(target))
if app_or_project == "app":
self.validate_name(os.path.basename(top_dir), "directory")
if not os.path.exists(top_dir):
raise CommandError("Destination directory '%s' does not "
"exist, please create it first." % top_dir)
extensions = tuple(handle_extensions(options["extensions"]))
extra_files = []
excluded_directories = [".git", "__pycache__"]
for file in options["files"]:
extra_files.extend(map(lambda x: x.strip(), file.split(",")))
if exclude := options.get("exclude"):
for directory in exclude:
excluded_directories.append(directory.strip())
if self.verbosity >= 2:
self.stdout.write(
"Rendering %s template files with extensions: %s" %
(app_or_project, ", ".join(extensions)))
self.stdout.write(
"Rendering %s template files with filenames: %s" %
(app_or_project, ", ".join(extra_files)))
base_name = "%s_name" % app_or_project
base_subdir = "%s_template" % app_or_project
base_directory = "%s_directory" % app_or_project
camel_case_name = "camel_case_%s_name" % app_or_project
camel_case_value = "".join(x for x in name.title() if x != "_")
context = Context(
{
**options,
base_name: name,
base_directory: top_dir,
camel_case_name: camel_case_value,
"docs_version": get_docs_version(),
"django_version": django.__version__,
},
autoescape=False,
)
# Setup a stub settings environment for template rendering
if not settings.configured:
settings.configure()
django.setup()
template_dir = self.handle_template(options["template"], base_subdir)
prefix_length = len(template_dir) + 1
for root, dirs, files in os.walk(template_dir):
path_rest = root[prefix_length:]
relative_dir = path_rest.replace(base_name, name)
if relative_dir:
target_dir = os.path.join(top_dir, relative_dir)
os.makedirs(target_dir, exist_ok=True)
for dirname in dirs[:]:
if "exclude" not in options:
if dirname.startswith(".") or dirname == "__pycache__":
dirs.remove(dirname)
elif dirname in excluded_directories:
dirs.remove(dirname)
for filename in files:
if filename.endswith((".pyo", ".pyc", ".py.class")):
# Ignore some files as they cause various breakages.
continue
old_path = os.path.join(root, filename)
new_path = os.path.join(top_dir, relative_dir,
filename.replace(base_name, name))
for old_suffix, new_suffix in self.rewrite_template_suffixes:
if new_path.endswith(old_suffix):
new_path = new_path[:-len(old_suffix)] + new_suffix
break # Only rewrite once
if os.path.exists(new_path):
raise CommandError(
"%s already exists. Overlaying %s %s into an existing "
"directory won't replace conflicting files." % (
new_path,
self.a_or_an,
app_or_project,
))
# Only render the Python files, as we don't want to
# accidentally render Django templates files
if new_path.endswith(extensions) or filename in extra_files:
with open(old_path, encoding="utf-8") as template_file:
content = template_file.read()
template = Engine().from_string(content)
content = template.render(context)
with open(new_path, "w", encoding="utf-8") as new_file:
new_file.write(content)
else:
shutil.copyfile(old_path, new_path)
if self.verbosity >= 2:
self.stdout.write("Creating %s" % new_path)
try:
shutil.copymode(old_path, new_path)
self.make_writeable(new_path)
except OSError:
self.stderr.write(
"Notice: Couldn't set permission bits on %s. You're "
"probably using an uncommon filesystem setup. No "
"problem." % new_path,
self.style.NOTICE,
)
if self.paths_to_remove:
if self.verbosity >= 2:
self.stdout.write("Cleaning up temporary files.")
for path_to_remove in self.paths_to_remove:
if os.path.isfile(path_to_remove):
os.remove(path_to_remove)
else:
shutil.rmtree(path_to_remove)
def handle(self,
app_or_project,
name,
target=None,
override=False,
fake=False,
**options):
self.app_or_project = app_or_project
self.paths_to_remove = []
self.verbosity = int(options.get('verbosity'))
self.validate_name(name, app_or_project)
if fake:
return
# if some directory is given, make sure it's nicely expanded
if target is None:
top_dir = path.join(os.getcwd(), name)
try:
os.makedirs(top_dir)
except OSError as e:
if e.errno == errno.EEXIST:
if not override:
raise CommandError("'%s' already exists" % top_dir)
else:
raise CommandError(e)
else:
top_dir = os.path.abspath(path.expanduser(target))
if not os.path.exists(top_dir):
raise CommandError(
"Destination directory '%s' does not "
"exist, please create it first." % top_dir
)
extensions = tuple(handle_extensions(options.get('extensions')))
extra_files = []
for file in options.get('files'):
extra_files.extend(map(lambda x: x.strip(), file.split(',')))
if self.verbosity >= 2:
self.stdout.write(
"Rendering %s template files with "
"extensions: %s\n" % (app_or_project, ', '.join(extensions))
)
self.stdout.write(
"Rendering %s template files with "
"filenames: %s\n" % (app_or_project, ', '.join(extra_files))
)
base_name = '%s_name' % app_or_project
base_subdir = '%s_template' % app_or_project
base_directory = '%s_directory' % app_or_project
if django.VERSION[-2] != 'final':
docs_version = 'dev'
else:
docs_version = '%d.%d' % django.VERSION[:2]
base_context = dict(
options, **{
base_name: name,
base_directory: top_dir,
'docs_version': get_docs_version(),
'django_version': django.__version__,
}
)
# Setup a stub settings environment for template rendering
from django.conf import settings
if not settings.configured:
settings.configure(DEBUG=True, TEMPLATE_DEBUG=True)
template_dir = self.handle_template(
options.get('template'), base_subdir
)
prefix_length = len(template_dir) + 1
for root, dirs, files in os.walk(template_dir):
path_rest = root[prefix_length:]
relative_dir = path_rest.replace(base_name, name)
global_context = {}
# Try to find context.yaml file in the current directory or
# parent directory(ies)
globals_file = os.path.join(root, 'globals.yaml')
while True:
globals_file = os.path.abspath(os.path.normpath(globals_file))
if not globals_file.startswith(template_dir):
break
if os.path.exists(globals_file):
# Read context
with open(globals_file, 'rb') as f:
globals_content = f.read()
global_context = yaml.safe_load(globals_content)
break
elif not os.path.dirname(globals_file):
break
else:
globals_file = os.path.join(
os.path.dirname(globals_file), os.pardir,
os.path.basename(globals_file)
)
for dirname in dirs[:]:
if dirname.startswith('.') or dirname == '__pycache__':
dirs.remove(dirname)
for filename in files:
if (
filename.endswith(('.pyo', '.pyc', '.py.class')) or
filename in ('globals.yaml', )
):
# Ignore some files as they cause various breakages.
continue
old_path = path.join(root, filename)
new_path = path.join(
top_dir, relative_dir, filename.replace(base_name, name)
)
if path.exists(new_path) and not override:
raise CommandError(
"%s already exists, overlaying a "
"project or app into an existing "
"directory won't replace conflicting "
"files" % new_path
)
with open(old_path, 'rb') as template_file:
content = template_file.read()
# Only get front matter from matching extensions
if filename.endswith(extensions) or filename in extra_files:
content = content.decode('utf-8')
content, extra_context = self.extract_front_matter(content)
context = dict(
base_context.items() + global_context.items(
) + extra_context.items()
)
else:
context = dict(
base_context.items() + global_context.items(
)
)
# Process context
out = self.preprocess_file(filename, context)
if out is False:
# If False is explicity returned we will not
# output this file
continue
elif isinstance(out, dict):
context = out
# Only render matching extensions
if filename.endswith(extensions) or filename in extra_files:
if self.verbosity >= 2:
self.stdout.write("Rendering %s\n" % old_path)
try:
template = Template(content)
content = template.render(
Context(
context,
autoescape=False
)
)
except Exception:
self.stdout.write("Rendering failed %s\n" % old_path)
raise
content = content.encode('utf-8')
if relative_dir:
dir_path = os.path.dirname(new_path)
if not path.exists(dir_path):
mkdirp(dir_path)
with open(new_path, 'wb') as new_file:
new_file.write(content)
if self.verbosity >= 2:
self.stdout.write("Creating %s\n" % new_path)
try:
shutil.copymode(old_path, new_path)
self.make_writeable(new_path)
except OSError:
self.stderr.write(
"Notice: Couldn't set permission bits on %s. You're "
"probably using an uncommon filesystem setup. No "
"problem." % new_path, self.style.NOTICE
)
if self.paths_to_remove:
if self.verbosity >= 2:
self.stdout.write("Cleaning up temporary files.\n")
for path_to_remove in self.paths_to_remove:
if path.isfile(path_to_remove):
os.remove(path_to_remove)
else:
shutil.rmtree(path_to_remove, onerror=rmtree_errorhandler)
def serialize(cls, value):
"""
Serializes the value to a string that's parsable by Python, along
with any needed imports to make that string work.
More advanced than repr() as it can encode things
like datetime.datetime.now.
"""
# FIXME: Ideally Promise would be reconstructible, but for now we
# use force_text on them and defer to the normal string serialization
# process.
if isinstance(value, Promise):
value = force_text(value)
elif isinstance(value, LazyObject):
# The unwrapped value is returned as the first item of the
# arguments tuple.
value = value.__reduce__()[1][0]
# Sequences
if isinstance(value, (frozenset, list, set, tuple)):
imports = set()
strings = []
for item in value:
item_string, item_imports = cls.serialize(item)
imports.update(item_imports)
strings.append(item_string)
if isinstance(value, set):
# Don't use the literal "{%s}" as it doesn't support empty set
format = "set([%s])"
elif isinstance(value, frozenset):
format = "frozenset([%s])"
elif isinstance(value, tuple):
# When len(value)==0, the empty tuple should be serialized as
# "()", not "(,)" because (,) is invalid Python syntax.
format = "(%s)" if len(value) != 1 else "(%s,)"
else:
format = "[%s]"
return format % (", ".join(strings)), imports
# Dictionaries
elif isinstance(value, dict):
imports = set()
strings = []
for k, v in sorted(value.items()):
k_string, k_imports = cls.serialize(k)
v_string, v_imports = cls.serialize(v)
imports.update(k_imports)
imports.update(v_imports)
strings.append((k_string, v_string))
return "{%s}" % (", ".join("%s: %s" % (k, v) for k, v in strings)), imports
# Datetimes
elif isinstance(value, datetime.datetime):
value_repr = cls.serialize_datetime(value)
imports = ["import datetime"]
if value.tzinfo is not None:
imports.append("from django.utils.timezone import utc")
return value_repr, set(imports)
# Dates
elif isinstance(value, datetime.date):
value_repr = repr(value)
if isinstance(value, datetime_safe.date):
value_repr = "datetime.%s" % value_repr
return value_repr, {"import datetime"}
# Times
elif isinstance(value, datetime.time):
value_repr = repr(value)
if isinstance(value, datetime_safe.time):
value_repr = "datetime.%s" % value_repr
return value_repr, {"import datetime"}
# Timedeltas
elif isinstance(value, datetime.timedelta):
return repr(value), {"import datetime"}
# Settings references
elif isinstance(value, SettingsReference):
return "secure.%s" % value.setting_name, {"from django.conf import secure"}
# Simple types
elif isinstance(value, float):
if math.isnan(value) or math.isinf(value):
return 'float("{}")'.format(value), set()
return repr(value), set()
elif isinstance(value, six.integer_types + (bool, type(None))):
return repr(value), set()
elif isinstance(value, six.binary_type):
value_repr = repr(value)
if six.PY2:
# Prepend the `b` prefix since we're importing unicode_literals
value_repr = 'b' + value_repr
return value_repr, set()
elif isinstance(value, six.text_type):
value_repr = repr(value)
if six.PY2:
# Strip the `u` prefix since we're importing unicode_literals
value_repr = value_repr[1:]
return value_repr, set()
# Decimal
elif isinstance(value, decimal.Decimal):
return repr(value), {"from decimal import Decimal"}
# Django fields
elif isinstance(value, models.Field):
attr_name, path, args, kwargs = value.deconstruct()
return cls.serialize_deconstructed(path, args, kwargs)
# Classes
elif isinstance(value, type):
special_cases = [
(models.Model, "models.Model", []),
]
for case, string, imports in special_cases:
if case is value:
return string, set(imports)
if hasattr(value, "__module__"):
module = value.__module__
if module == six.moves.builtins.__name__:
return value.__name__, set()
else:
return "%s.%s" % (module, value.__name__), {"import %s" % module}
elif isinstance(value, models.manager.BaseManager):
as_manager, manager_path, qs_path, args, kwargs = value.deconstruct()
if as_manager:
name, imports = cls._serialize_path(qs_path)
return "%s.as_manager()" % name, imports
else:
return cls.serialize_deconstructed(manager_path, args, kwargs)
elif isinstance(value, Operation):
string, imports = OperationWriter(value, indentation=0).serialize()
# Nested operation, trailing comma is handled in upper OperationWriter._write()
return string.rstrip(','), imports
elif isinstance(value, functools.partial):
imports = {'import functools'}
# Serialize functools.partial() arguments
func_string, func_imports = cls.serialize(value.func)
args_string, args_imports = cls.serialize(value.args)
keywords_string, keywords_imports = cls.serialize(value.keywords)
# Add any imports needed by arguments
imports.update(func_imports)
imports.update(args_imports)
imports.update(keywords_imports)
return (
"functools.partial(%s, *%s, **%s)" % (
func_string, args_string, keywords_string,
),
imports,
)
# Anything that knows how to deconstruct itself.
elif hasattr(value, 'deconstruct'):
return cls.serialize_deconstructed(*value.deconstruct())
# Functions
elif isinstance(value, (types.FunctionType, types.BuiltinFunctionType)):
# @classmethod?
if getattr(value, "__self__", None) and isinstance(value.__self__, type):
klass = value.__self__
module = klass.__module__
return "%s.%s.%s" % (module, klass.__name__, value.__name__), {"import %s" % module}
# Further error checking
if value.__name__ == '<lambda>':
raise ValueError("Cannot serialize function: lambda")
if value.__module__ is None:
raise ValueError("Cannot serialize function %r: No module" % value)
# Python 3 is a lot easier, and only uses this branch if it's not local.
if getattr(value, "__qualname__", None) and getattr(value, "__module__", None):
if "<" not in value.__qualname__: # Qualname can include <locals>
return "%s.%s" % (value.__module__, value.__qualname__), {"import %s" % value.__module__}
# Python 2/fallback version
module_name = value.__module__
# Make sure it's actually there and not an unbound method
module = import_module(module_name)
if not hasattr(module, value.__name__):
raise ValueError(
"Could not find function %s in %s.\n"
"Please note that due to Python 2 limitations, you cannot "
"serialize unbound method functions (e.g. a method "
"declared and used in the same class body). Please move "
"the function into the main module body to use migrations.\n"
"For more information, see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#serializing-values"
% (value.__name__, module_name, get_docs_version()))
# Needed on Python 2 only
if module_name == '__builtin__':
return value.__name__, set()
return "%s.%s" % (module_name, value.__name__), {"import %s" % module_name}
# Other iterables
elif isinstance(value, collections.Iterable):
imports = set()
strings = []
for item in value:
item_string, item_imports = cls.serialize(item)
imports.update(item_imports)
strings.append(item_string)
# When len(strings)==0, the empty iterable should be serialized as
# "()", not "(,)" because (,) is invalid Python syntax.
format = "(%s)" if len(strings) != 1 else "(%s,)"
return format % (", ".join(strings)), imports
# Compiled regex
elif isinstance(value, (COMPILED_REGEX_TYPE, RegexObject)):
imports = {"import re"}
regex_pattern, pattern_imports = cls.serialize(value.pattern)
regex_flags, flag_imports = cls.serialize(value.flags)
imports.update(pattern_imports)
imports.update(flag_imports)
args = [regex_pattern]
if value.flags:
args.append(regex_flags)
return "re.compile(%s)" % ', '.join(args), imports
# Uh oh.
else:
raise ValueError(
"Cannot serialize: %r\nThere are some values Django cannot serialize into "
"migration files.\nFor more, see https://docs.djangoproject.com/en/%s/"
"topics/migrations/#migration-serializing" % (value, get_docs_version())
)
def handle(self, **options):
self.verbosity = options['verbosity']
self.interactive = options['interactive']
app_label = options['app_label']
start_migration_name = options['start_migration_name']
migration_name = options['migration_name']
no_optimize = options['no_optimize']
squashed_name = options['squashed_name']
include_header = options['include_header']
# Validate app_label.
try:
apps.get_app_config(app_label)
except LookupError as err:
raise CommandError(str(err))
# Load the current graph state, check the BLOG and migration they asked for exists
loader = MigrationLoader(connections[DEFAULT_DB_ALIAS])
if app_label not in loader.migrated_apps:
raise CommandError(
"App '%s' does not have migrations (so squashmigrations on "
"it makes no sense)" % app_label)
migration = self.find_migration(loader, app_label, migration_name)
# Work out the list of predecessor migrations
migrations_to_squash = [
loader.get_migration(al, mn)
for al, mn in loader.graph.forwards_plan((migration.app_label,
migration.name))
if al == migration.app_label
]
if start_migration_name:
start_migration = self.find_migration(loader, app_label,
start_migration_name)
start = loader.get_migration(start_migration.app_label,
start_migration.name)
try:
start_index = migrations_to_squash.index(start)
migrations_to_squash = migrations_to_squash[start_index:]
except ValueError:
raise CommandError(
"The migration '%s' cannot be found. Maybe it comes after "
"the migration '%s'?\n"
"Have a look at:\n"
" python manage.py showmigrations %s\n"
"to debug this issue." %
(start_migration, migration, app_label))
# Tell them what we're doing and optionally ask if we should proceed
if self.verbosity > 0 or self.interactive:
self.stdout.write(
self.style.MIGRATE_HEADING(
"Will squash the following migrations:"))
for migration in migrations_to_squash:
self.stdout.write(" - %s" % migration.name)
if self.interactive:
answer = None
while not answer or answer not in "yn":
answer = input("Do you wish to proceed? [yN] ")
if not answer:
answer = "n"
break
else:
answer = answer[0].lower()
if answer != "y":
return
# Load the operations from all those migrations and concat together,
# along with collecting external dependencies and detecting
# double-squashing
operations = []
dependencies = set()
# We need to take all dependencies from the first migration in the list
# as it may be 0002 depending on 0001
first_migration = True
for smigration in migrations_to_squash:
if smigration.replaces:
raise CommandError(
"You cannot squash squashed migrations! Please transition "
"it to a normal migration first: "
"https://docs.djangoproject.com/en/%s/topics/migrations/#squashing-migrations"
% get_docs_version())
operations.extend(smigration.operations)
for dependency in smigration.dependencies:
if isinstance(dependency, SwappableTuple):
if settings.AUTH_USER_MODEL == dependency.setting:
dependencies.add(("__setting__", "AUTH_USER_MODEL"))
else:
dependencies.add(dependency)
elif dependency[0] != smigration.app_label or first_migration:
dependencies.add(dependency)
first_migration = False
if no_optimize:
if self.verbosity > 0:
self.stdout.write(
self.style.MIGRATE_HEADING("(Skipping optimization.)"))
new_operations = operations
else:
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("Optimizing..."))
optimizer = MigrationOptimizer()
new_operations = optimizer.optimize(operations,
migration.app_label)
if self.verbosity > 0:
if len(new_operations) == len(operations):
self.stdout.write(" No optimizations possible.")
else:
self.stdout.write(
" Optimized from %s operations to %s operations." %
(len(operations), len(new_operations)))
# Work out the value of replaces (any squashed ones we're re-squashing)
# need to feed their replaces into ours
replaces = []
for migration in migrations_to_squash:
if migration.replaces:
replaces.extend(migration.replaces)
else:
replaces.append((migration.app_label, migration.name))
# Make a new migration with those operations
subclass = type(
"Migration", (migrations.Migration, ), {
"dependencies": dependencies,
"operations": new_operations,
"replaces": replaces,
})
if start_migration_name:
if squashed_name:
# Use the name from --squashed-name.
prefix, _ = start_migration.name.split('_', 1)
name = '%s_%s' % (prefix, squashed_name)
else:
# Generate a name.
name = '%s_squashed_%s' % (start_migration.name,
migration.name)
new_migration = subclass(name, app_label)
else:
name = '0001_%s' % (squashed_name
or 'squashed_%s' % migration.name)
new_migration = subclass(name, app_label)
new_migration.initial = True
# Write out the new migration file
writer = MigrationWriter(new_migration, include_header)
with open(writer.path, "w", encoding='utf-8') as fh:
fh.write(writer.as_string())
if self.verbosity > 0:
self.stdout.write(
self.style.MIGRATE_HEADING(
'Created new squashed migration %s' % writer.path) + '\n'
' You should commit this migration but leave the old ones in place;\n'
' the new migration will be used for new installs. Once you are sure\n'
' all instances of the codebase have applied the migrations you squashed,\n'
' you can delete them.')
if writer.needs_manual_porting:
self.stdout.write(
self.style.MIGRATE_HEADING('Manual porting required') +
'\n'
' Your migrations contained functions that must be manually copied over,\n'
' as we could not safely copy their implementation.\n'
' See the comment at the top of the squashed migration for details.'
)
def serializer_factory(value):
from django.db.migrations.writer import SettingsReference
if isinstance(value, Promise):
value = str(value)
elif isinstance(value, LazyObject):
# The unwrapped value is returned as the first item of the arguments
# tuple.
value = value.__reduce__()[1][0]
if isinstance(value, models.Field):
return ModelFieldSerializer(value)
if isinstance(value, models.manager.BaseManager):
return ModelManagerSerializer(value)
if isinstance(value, Operation):
return OperationSerializer(value)
if isinstance(value, type):
return TypeSerializer(value)
# Anything that knows how to deconstruct itself.
if hasattr(value, 'deconstruct'):
return DeconstructableSerializer(value)
# Unfortunately some of these are order-dependent.
if isinstance(value, frozenset):
return FrozensetSerializer(value)
if isinstance(value, list):
return SequenceSerializer(value)
if isinstance(value, set):
return SetSerializer(value)
if isinstance(value, tuple):
return TupleSerializer(value)
if isinstance(value, dict):
return DictionarySerializer(value)
if isinstance(value, enum.Enum):
return EnumSerializer(value)
if isinstance(value, datetime.datetime):
return DatetimeSerializer(value)
if isinstance(value, datetime.date):
return DateSerializer(value)
if isinstance(value, datetime.time):
return TimeSerializer(value)
if isinstance(value, datetime.timedelta):
return TimedeltaSerializer(value)
if isinstance(value, SettingsReference):
return SettingsReferenceSerializer(value)
if isinstance(value, float):
return FloatSerializer(value)
if isinstance(value, (bool, int, type(None), bytes, str)):
return BaseSimpleSerializer(value)
if isinstance(value, decimal.Decimal):
return DecimalSerializer(value)
if isinstance(value, (functools.partial, functools.partialmethod)):
return FunctoolsPartialSerializer(value)
if isinstance(value, (types.FunctionType, types.BuiltinFunctionType, types.MethodType)):
return FunctionTypeSerializer(value)
if isinstance(value, collections.Iterable):
return IterableSerializer(value)
if isinstance(value, (COMPILED_REGEX_TYPE, RegexObject)):
return RegexSerializer(value)
if isinstance(value, uuid.UUID):
return UUIDSerializer(value)
raise ValueError(
"Cannot serialize: %r\nThere are some values Django cannot serialize into "
"migration files.\nFor more, see https://docs.djangoproject.com/en/%s/"
"topics/migrations/#migration-serializing" % (value, get_docs_version())
)
#source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = '<a href="https://www.bounca.org">BounCA</a>' copyright = '2016, Repleo, Amstelveen, Holland' author = 'Jeroen Arnoldus' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = str(get_docs_version(VERSION)) # The full version, including alpha/beta/rc tags. release = str(get_version(VERSION)) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y'
def handle(self, app_or_project, name, target=None, **options):
self.app_or_project = app_or_project
self.paths_to_remove = []
self.verbosity = options['verbosity']
self.validate_name(name, app_or_project)
# if some directory is given, make sure it's nicely expanded
if target is None:
top_dir = path.join(os.getcwd(), name)
try:
os.makedirs(top_dir)
except OSError as e:
if e.errno == errno.EEXIST:
message = "'%s' already exists" % top_dir
else:
message = e
raise CommandError(message)
else:
top_dir = os.path.abspath(path.expanduser(target))
if not os.path.exists(top_dir):
raise CommandError("Destination directory '%s' does not "
"exist, please create it first." % top_dir)
extensions = tuple(handle_extensions(options['extensions']))
extra_files = []
for file in options['files']:
extra_files.extend(map(lambda x: x.strip(), file.split(',')))
if self.verbosity >= 2:
self.stdout.write("Rendering %s template files with "
"extensions: %s\n" %
(app_or_project, ', '.join(extensions)))
self.stdout.write("Rendering %s template files with "
"filenames: %s\n" %
(app_or_project, ', '.join(extra_files)))
base_name = '%s_name' % app_or_project
base_subdir = '%s_template' % app_or_project
base_directory = '%s_directory' % app_or_project
camel_case_name = 'camel_case_%s_name' % app_or_project
camel_case_value = ''.join(x for x in name.title() if x != '_')
context = Context(dict(options, **{
base_name: name,
base_directory: top_dir,
camel_case_name: camel_case_value,
'docs_version': get_docs_version(),
'django_version': django.__version__,
'unicode_literals': '' if six.PY3 else 'from __future__ import unicode_literals\n\n',
}), autoescape=False)
# Setup a stub settings environment for template rendering
from django.conf import settings
if not settings.configured:
settings.configure()
template_dir = self.handle_template(options['template'],
base_subdir)
prefix_length = len(template_dir) + 1
for root, dirs, files in os.walk(template_dir):
path_rest = root[prefix_length:]
relative_dir = path_rest.replace(base_name, name)
if relative_dir:
target_dir = path.join(top_dir, relative_dir)
if not path.exists(target_dir):
os.mkdir(target_dir)
for dirname in dirs[:]:
if dirname.startswith('.') or dirname == '__pycache__':
dirs.remove(dirname)
for filename in files:
if filename.endswith(('.pyo', '.pyc', '.py.class')):
# Ignore some files as they cause various breakages.
continue
old_path = path.join(root, filename)
new_path = path.join(top_dir, relative_dir,
filename.replace(base_name, name))
for old_suffix, new_suffix in self.rewrite_template_suffixes:
if new_path.endswith(old_suffix):
new_path = new_path[:-len(old_suffix)] + new_suffix
break # Only rewrite once
if path.exists(new_path):
raise CommandError("%s already exists, overlaying a "
"project or app into an existing "
"directory won't replace conflicting "
"files" % new_path)
# Only render the Python files, as we don't want to
# accidentally render Django templates files
with open(old_path, 'rb') as template_file:
content = template_file.read()
if new_path.endswith(extensions) or filename in extra_files:
content = content.decode('utf-8')
template = Engine().from_string(content)
content = template.render(context)
content = content.encode('utf-8')
with open(new_path, 'wb') as new_file:
new_file.write(content)
if self.verbosity >= 2:
self.stdout.write("Creating %s\n" % new_path)
try:
shutil.copymode(old_path, new_path)
self.make_writeable(new_path)
except OSError:
self.stderr.write(
"Notice: Couldn't set permission bits on %s. You're "
"probably using an uncommon filesystem setup. No "
"problem." % new_path, self.style.NOTICE)
if self.paths_to_remove:
if self.verbosity >= 2:
self.stdout.write("Cleaning up temporary files.\n")
for path_to_remove in self.paths_to_remove:
if path.isfile(path_to_remove):
os.remove(path_to_remove)
else:
shutil.rmtree(path_to_remove)
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
# Want to get everything from the 'normal' models package.
from django.db.models import * # NOQA
from django.utils.version import get_docs_version
from django.contrib.gis.geos import HAS_GEOS
if not HAS_GEOS:
raise ImproperlyConfigured(
"GEOS is required and has not been detected. Are you sure it is installed? "
"See also https://docs.djangoproject.com/en/%s/ref/contrib/gis/install/geolibs/" % get_docs_version())
# Geographic aggregate functions
from django.contrib.gis.db.models.aggregates import * # NOQA
# The GeoManager
from django.contrib.gis.db.models.manager import GeoManager # NOQA
# The geographic-enabled fields.
from django.contrib.gis.db.models.fields import ( # NOQA
GeometryField, PointField, LineStringField, PolygonField,
MultiPointField, MultiLineStringField, MultiPolygonField,
GeometryCollectionField)
def handle(self, **options):
self.verbosity = options['verbosity']
self.interactive = options['interactive']
app_label = options['app_label']
start_migration_name = options['start_migration_name']
migration_name = options['migration_name']
no_optimize = options['no_optimize']
# Load the current graph state, check the app and migration they asked for exists
loader = MigrationLoader(connections[DEFAULT_DB_ALIAS])
if app_label not in loader.migrated_apps:
raise CommandError(
"App '%s' does not have migrations (so squashmigrations on "
"it makes no sense)" % app_label
)
migration = self.find_migration(loader, app_label, migration_name)
# Work out the list of predecessor migrations
migrations_to_squash = [
loader.get_migration(al, mn)
for al, mn in loader.graph.forwards_plan((migration.app_label, migration.name))
if al == migration.app_label
]
if start_migration_name:
start_migration = self.find_migration(loader, app_label, start_migration_name)
start = loader.get_migration(start_migration.app_label, start_migration.name)
try:
start_index = migrations_to_squash.index(start)
migrations_to_squash = migrations_to_squash[start_index:]
except ValueError:
raise CommandError(
"The migration '%s' cannot be found. Maybe it comes after "
"the migration '%s'?\n"
"Have a look at:\n"
" python manage.py showmigrations %s\n"
"to debug this issue." % (start_migration, migration, app_label)
)
# Tell them what we're doing and optionally ask if we should proceed
if self.verbosity > 0 or self.interactive:
self.stdout.write(self.style.MIGRATE_HEADING("Will squash the following migrations:"))
for migration in migrations_to_squash:
self.stdout.write(" - %s" % migration.name)
if self.interactive:
answer = None
while not answer or answer not in "yn":
answer = six.moves.input("Do you wish to proceed? [yN] ")
if not answer:
answer = "n"
break
else:
answer = answer[0].lower()
if answer != "y":
return
# Load the operations from all those migrations and concat together,
# along with collecting external dependencies and detecting
# double-squashing
operations = []
dependencies = set()
# We need to take all dependencies from the first migration in the list
# as it may be 0002 depending on 0001
first_migration = True
for smigration in migrations_to_squash:
if smigration.replaces:
raise CommandError(
"You cannot squash squashed migrations! Please transition "
"it to a normal migration first: "
"https://docs.djangoproject.com/en/%s/topics/migrations/#squashing-migrations" % get_docs_version()
)
operations.extend(smigration.operations)
for dependency in smigration.dependencies:
if isinstance(dependency, SwappableTuple):
if settings.AUTH_USER_MODEL == dependency.setting:
dependencies.add(("__setting__", "AUTH_USER_MODEL"))
else:
dependencies.add(dependency)
elif dependency[0] != smigration.app_label or first_migration:
dependencies.add(dependency)
first_migration = False
if no_optimize:
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("(Skipping optimization.)"))
new_operations = operations
else:
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("Optimizing..."))
optimizer = MigrationOptimizer()
new_operations = optimizer.optimize(operations, migration.app_label)
if self.verbosity > 0:
if len(new_operations) == len(operations):
self.stdout.write(" No optimizations possible.")
else:
self.stdout.write(
" Optimized from %s operations to %s operations." %
(len(operations), len(new_operations))
)
# Work out the value of replaces (any squashed ones we're re-squashing)
# need to feed their replaces into ours
replaces = []
for migration in migrations_to_squash:
if migration.replaces:
replaces.extend(migration.replaces)
else:
replaces.append((migration.app_label, migration.name))
# Make a new migration with those operations
subclass = type("Migration", (migrations.Migration, ), {
"dependencies": dependencies,
"operations": new_operations,
"replaces": replaces,
})
if start_migration_name:
new_migration = subclass("%s_squashed_%s" % (start_migration.name, migration.name), app_label)
else:
new_migration = subclass("0001_squashed_%s" % migration.name, app_label)
new_migration.initial = True
# Write out the new migration file
writer = MigrationWriter(new_migration)
with open(writer.path, "wb") as fh:
fh.write(writer.as_string())
if self.verbosity > 0:
self.stdout.write(self.style.MIGRATE_HEADING("Created new squashed migration %s" % writer.path))
self.stdout.write(" You should commit this migration but leave the old ones in place;")
self.stdout.write(" the new migration will be used for new installs. Once you are sure")
self.stdout.write(" all instances of the codebase have applied the migrations you squashed,")
self.stdout.write(" you can delete them.")
if writer.needs_manual_porting:
self.stdout.write(self.style.MIGRATE_HEADING("Manual porting required"))
self.stdout.write(" Your migrations contained functions that must be manually copied over,")
self.stdout.write(" as we could not safely copy their implementation.")
self.stdout.write(" See the comment at the top of the squashed migration for details.")
def serialize(cls, value):
"""
Serializes the value to a string that's parsable by Python, along
with any needed imports to make that string work.
More advanced than repr() as it can encode things
like datetime.datetime.now.
"""
# FIXME: Ideally Promise would be reconstructible, but for now we
# use force_text on them and defer to the normal string serialization
# process.
if isinstance(value, Promise):
value = force_text(value)
# Sequences
if isinstance(value, (frozenset, list, set, tuple)):
imports = set()
strings = []
for item in value:
item_string, item_imports = cls.serialize(item)
imports.update(item_imports)
strings.append(item_string)
if isinstance(value, set):
# Don't use the literal "{%s}" as it doesn't support empty set
format = "set([%s])"
elif isinstance(value, frozenset):
format = "frozenset([%s])"
elif isinstance(value, tuple):
# When len(value)==0, the empty tuple should be serialized as
# "()", not "(,)" because (,) is invalid Python syntax.
format = "(%s)" if len(value) != 1 else "(%s,)"
else:
format = "[%s]"
return format % (", ".join(strings)), imports
# Dictionaries
elif isinstance(value, dict):
imports = set()
strings = []
for k, v in sorted(value.items()):
k_string, k_imports = cls.serialize(k)
v_string, v_imports = cls.serialize(v)
imports.update(k_imports)
imports.update(v_imports)
strings.append((k_string, v_string))
return "{%s}" % (", ".join("%s: %s" % (k, v)
for k, v in strings)), imports
# Datetimes
elif isinstance(value, datetime.datetime):
value_repr = cls.serialize_datetime(value)
imports = ["import datetime"]
if value.tzinfo is not None:
imports.append("from django.utils.timezone import utc")
return value_repr, set(imports)
# Dates
elif isinstance(value, datetime.date):
value_repr = repr(value)
if isinstance(value, datetime_safe.date):
value_repr = "datetime.%s" % value_repr
return value_repr, {"import datetime"}
# Times
elif isinstance(value, datetime.time):
value_repr = repr(value)
if isinstance(value, datetime_safe.time):
value_repr = "datetime.%s" % value_repr
return value_repr, {"import datetime"}
# Timedeltas
elif isinstance(value, datetime.timedelta):
return repr(value), {"import datetime"}
# Settings references
elif isinstance(value, SettingsReference):
return "settings.%s" % value.setting_name, {
"from django.conf import settings"
}
# Simple types
elif isinstance(value, float):
if math.isnan(value) or math.isinf(value):
return 'float("{}")'.format(value), set()
return repr(value), set()
elif isinstance(value, six.integer_types + (bool, type(None))):
return repr(value), set()
elif isinstance(value, six.binary_type):
value_repr = repr(value)
if six.PY2:
# Prepend the `b` prefix since we're importing unicode_literals
value_repr = 'b' + value_repr
return value_repr, set()
elif isinstance(value, six.text_type):
value_repr = repr(value)
if six.PY2:
# Strip the `u` prefix since we're importing unicode_literals
value_repr = value_repr[1:]
return value_repr, set()
# Decimal
elif isinstance(value, decimal.Decimal):
return repr(value), {"from decimal import Decimal"}
# Django fields
elif isinstance(value, models.Field):
attr_name, path, args, kwargs = value.deconstruct()
return cls.serialize_deconstructed(path, args, kwargs)
# Classes
elif isinstance(value, type):
special_cases = [
(models.Model, "models.Model", []),
]
for case, string, imports in special_cases:
if case is value:
return string, set(imports)
if hasattr(value, "__module__"):
module = value.__module__
if module == six.moves.builtins.__name__:
return value.__name__, set()
else:
return "%s.%s" % (module, value.__name__), {
"import %s" % module
}
elif isinstance(value, models.manager.BaseManager):
as_manager, manager_path, qs_path, args, kwargs = value.deconstruct(
)
if as_manager:
name, imports = cls._serialize_path(qs_path)
return "%s.as_manager()" % name, imports
else:
return cls.serialize_deconstructed(manager_path, args, kwargs)
elif isinstance(value, Operation):
string, imports = OperationWriter(value, indentation=0).serialize()
# Nested operation, trailing comma is handled in upper OperationWriter._write()
return string.rstrip(','), imports
elif isinstance(value, functools.partial):
imports = {'import functools'}
# Serialize functools.partial() arguments
func_string, func_imports = cls.serialize(value.func)
args_string, args_imports = cls.serialize(value.args)
keywords_string, keywords_imports = cls.serialize(value.keywords)
# Add any imports needed by arguments
imports.update(func_imports)
imports.update(args_imports)
imports.update(keywords_imports)
return (
"functools.partial(%s, *%s, **%s)" % (
func_string,
args_string,
keywords_string,
),
imports,
)
# Anything that knows how to deconstruct itself.
elif hasattr(value, 'deconstruct'):
return cls.serialize_deconstructed(*value.deconstruct())
# Functions
elif isinstance(value,
(types.FunctionType, types.BuiltinFunctionType)):
# @classmethod?
if getattr(value, "__self__", None) and isinstance(
value.__self__, type):
klass = value.__self__
module = klass.__module__
return "%s.%s.%s" % (module, klass.__name__, value.__name__), {
"import %s" % module
}
# Further error checking
if value.__name__ == '<lambda>':
raise ValueError("Cannot serialize function: lambda")
if value.__module__ is None:
raise ValueError("Cannot serialize function %r: No module" %
value)
# Python 3 is a lot easier, and only uses this branch if it's not local.
if getattr(value, "__qualname__", None) and getattr(
value, "__module__", None):
if "<" not in value.__qualname__: # Qualname can include <locals>
return "%s.%s" % (value.__module__, value.__qualname__), {
"import %s" % value.__module__
}
# Python 2/fallback version
module_name = value.__module__
# Make sure it's actually there and not an unbound method
module = import_module(module_name)
if not hasattr(module, value.__name__):
raise ValueError(
"Could not find function %s in %s.\n"
"Please note that due to Python 2 limitations, you cannot "
"serialize unbound method functions (e.g. a method "
"declared and used in the same class body). Please move "
"the function into the main module body to use migrations.\n"
"For more information, see "
"https://docs.djangoproject.com/en/%s/topics/migrations/#serializing-values"
% (value.__name__, module_name, get_docs_version()))
# Needed on Python 2 only
if module_name == '__builtin__':
return value.__name__, set()
return "%s.%s" % (module_name, value.__name__), {
"import %s" % module_name
}
# Other iterables
elif isinstance(value, collections.Iterable):
imports = set()
strings = []
for item in value:
item_string, item_imports = cls.serialize(item)
imports.update(item_imports)
strings.append(item_string)
# When len(strings)==0, the empty iterable should be serialized as
# "()", not "(,)" because (,) is invalid Python syntax.
format = "(%s)" if len(strings) != 1 else "(%s,)"
return format % (", ".join(strings)), imports
# Compiled regex
elif isinstance(value, (COMPILED_REGEX_TYPE, RegexObject)):
imports = {"import re"}
regex_pattern, pattern_imports = cls.serialize(value.pattern)
regex_flags, flag_imports = cls.serialize(value.flags)
imports.update(pattern_imports)
imports.update(flag_imports)
args = [regex_pattern]
if value.flags:
args.append(regex_flags)
return "re.compile(%s)" % ', '.join(args), imports
# Uh oh.
else:
raise ValueError(
"Cannot serialize: %r\nThere are some values Django cannot serialize into "
"migration files.\nFor more, see https://docs.djangoproject.com/en/%s/"
"topics/migrations/#migration-serializing" %
(value, get_docs_version()))
from __future__ import unicode_literals
def serializer_factory(value):
from django.db.migrations.writer import SettingsReference
if isinstance(value, Promise):
value = str(value)
elif isinstance(value, LazyObject):
# The unwrapped value is returned as the first item of the arguments
# tuple.
value = value.__reduce__()[1][0]
if isinstance(value, models.Field):
return ModelFieldSerializer(value)
if isinstance(value, models.manager.BaseManager):
return ModelManagerSerializer(value)
if isinstance(value, Operation):
return OperationSerializer(value)
if isinstance(value, type):
return TypeSerializer(value)
# Anything that knows how to deconstruct itself.
if hasattr(value, 'deconstruct'):
return DeconstructableSerializer(value)
# Unfortunately some of these are order-dependent.
if isinstance(value, frozenset):
return FrozensetSerializer(value)
if isinstance(value, list):
return SequenceSerializer(value)
if isinstance(value, set):
return SetSerializer(value)
if isinstance(value, tuple):
return TupleSerializer(value)
if isinstance(value, dict):
return DictionarySerializer(value)
if isinstance(value, enum.Enum):
return EnumSerializer(value)
if isinstance(value, datetime.datetime):
return DatetimeSerializer(value)
if isinstance(value, datetime.date):
return DateSerializer(value)
if isinstance(value, datetime.time):
return TimeSerializer(value)
if isinstance(value, datetime.timedelta):
return TimedeltaSerializer(value)
if isinstance(value, SettingsReference):
return SettingsReferenceSerializer(value)
if isinstance(value, float):
return FloatSerializer(value)
if isinstance(value, (bool, int, type(None), bytes, str)):
return BaseSimpleSerializer(value)
if isinstance(value, decimal.Decimal):
return DecimalSerializer(value)
if isinstance(value, functools.partial):
return FunctoolsPartialSerializer(value)
if isinstance(
value,
(types.FunctionType, types.BuiltinFunctionType, types.MethodType)):
return FunctionTypeSerializer(value)
if isinstance(value, collections.Iterable):
return IterableSerializer(value)
if isinstance(value, (COMPILED_REGEX_TYPE, RegexObject)):
return RegexSerializer(value)
if isinstance(value, uuid.UUID):
return UUIDSerializer(value)
raise ValueError(
"Cannot serialize: %r\nThere are some values Django cannot serialize into "
"migration files.\nFor more, see https://docs.djangoproject.com/en/%s/"
"topics/migrations/#migration-serializing" %
(value, get_docs_version()))
import cgi
