def get_sql_evolution_check_for_changed_field_name(klass, old_table_name, style):
from django.db import get_introspection_module, connection
ops, introspection = get_operations_and_introspection_classes(style)
cursor = connection.cursor()
opts = klass._meta
output = []
db_table = klass._meta.db_table
if old_table_name:
db_table = old_table_name
for f in opts.fields:
existing_fields = introspection.get_columns(cursor,db_table)
if f.column in existing_fields:
old_col = f.column
elif f.aka and len(set(f.aka).intersection(set(existing_fields)))==1:
old_col = set(f.aka).intersection(set(existing_fields)).pop()
elif f.aka and len(set(f.aka).intersection(set(existing_fields)))>1:
details = 'column "%s" of table "%s"' % (f.column, klass._meta.db_table)
raise MultipleRenamesPossibleException("when renamed " + details)
else:
continue
if old_col != f.column:
col_type = f.db_type()
col_type_def = style.SQL_COLTYPE(col_type)
if col_type is not None:
col_def = style.SQL_COLTYPE(col_type) +' '+ style.SQL_KEYWORD('%sNULL' % (not f.null and 'NOT ' or ''))
if f.unique and not f.primary_key:
col_def += style.SQL_KEYWORD(' UNIQUE')
if f.primary_key:
col_def += style.SQL_KEYWORD(' PRIMARY KEY')
output.extend( ops.get_change_column_name_sql( klass._meta.db_table, get_introspection_module().get_indexes(cursor,db_table), old_col, f.column, col_type_def, f ) )
return output
def get_sql_evolution_rebuild_table(klass, old_table_name, style):
from django.db import get_introspection_module, connection
ops, introspection = get_operations_and_introspection_classes(style)
cursor = connection.cursor()
opts = klass._meta
output = []
db_table = klass._meta.db_table
if old_table_name:
db_table = old_table_name
existing_fields = introspection.get_columns(cursor,db_table)
renamed_fields = {}
for f in opts.fields:
if f.column in existing_fields:
old_col = f.column
elif f.aka and len(set(f.aka).intersection(set(existing_fields)))==1:
old_col = set(f.aka).intersection(set(existing_fields)).pop()
elif f.aka and len(set(f.aka).intersection(set(existing_fields)))>1:
details = 'column "%s" of table "%s"' % (f.column, klass._meta.db_table)
raise MultipleRenamesPossibleException("when renamed " + details)
else:
continue
if old_col != f.column:
renamed_fields[f.column] = old_col
output.extend( ops.get_rebuild_table_sql( klass._meta.db_table, get_introspection_module().get_indexes(cursor,db_table), existing_fields, renamed_fields) )
return output
def __init__(self, app_models, options):
self.app_models = app_models
self.options = options
self.dense = options.get('dense_output', False)
try:
self.introspection = connection.introspection
except AttributeError:
from django.db import get_introspection_module
self.introspection = get_introspection_module()
self.cursor = connection.cursor()
self.django_tables = self.get_django_tables(
options.get('only_existing', True))
self.db_tables = self.introspection.get_table_list(self.cursor)
self.differences = []
self.unknown_db_fields = {}
self.DIFF_SQL = {
'error': self.SQL_ERROR,
'comment': self.SQL_COMMENT,
'table-missing-in-db': self.SQL_TABLE_MISSING_IN_DB,
'field-missing-in-db': self.SQL_FIELD_MISSING_IN_DB,
'field-missing-in-model': self.SQL_FIELD_MISSING_IN_MODEL,
'index-missing-in-db': self.SQL_INDEX_MISSING_IN_DB,
'index-missing-in-model': self.SQL_INDEX_MISSING_IN_MODEL,
'unique-missing-in-db': self.SQL_UNIQUE_MISSING_IN_DB,
'unique-missing-in-model': self.SQL_UNIQUE_MISSING_IN_MODEL,
'field-type-differ': self.SQL_FIELD_TYPE_DIFFER,
'field-parameter-differ': self.SQL_FIELD_PARAMETER_DIFFER,
'notnull-differ': self.SQL_NOTNULL_DIFFER,
}
def __init__(self, app_models, options):
self.app_models = app_models
self.options = options
self.dense = options.get('dense_output', False)
try:
self.introspection = connection.introspection
except AttributeError:
from django.db import get_introspection_module
self.introspection = get_introspection_module()
self.cursor = connection.cursor()
self.django_tables = self.get_django_tables(options.get('only_existing', True))
self.db_tables = self.introspection.get_table_list(self.cursor)
self.differences = []
self.unknown_db_fields = {}
self.DIFF_SQL = {
'comment': self.SQL_COMMENT,
'table-missing-in-db': self.SQL_TABLE_MISSING_IN_DB,
'field-missing-in-db': self.SQL_FIELD_MISSING_IN_DB,
'field-missing-in-model': self.SQL_FIELD_MISSING_IN_MODEL,
'index-missing-in-db': self.SQL_INDEX_MISSING_IN_DB,
'index-missing-in-model': self.SQL_INDEX_MISSING_IN_MODEL,
'unique-missing-in-db': self.SQL_UNIQUE_MISSING_IN_DB,
'unique-missing-in-model': self.SQL_UNIQUE_MISSING_IN_MODEL,
'field-type-differ': self.SQL_FIELD_TYPE_DIFFER,
'field-parameter-differ': self.SQL_FIELD_PARAMETER_DIFFER,
}
def __init__(self, app_models, options):
self.has_differences = None
self.app_models = app_models
self.options = options
self.dense = options.get('dense_output', False)
try:
self.introspection = connection.introspection
except AttributeError:
from django.db import get_introspection_module
self.introspection = get_introspection_module()
self.cursor = connection.cursor()
self.django_tables = self.get_django_tables(options.get('only_existing', True))
self.db_tables = self.introspection.get_table_list(self.cursor)
if django.VERSION[:2] >= (1, 8):
# TODO: We are losing information about tables which are views here
self.db_tables = [table_info.name for table_info in self.db_tables]
self.differences = []
self.unknown_db_fields = {}
self.new_db_fields = set()
self.null = {}
self.unsigned = set()
self.DIFF_SQL = {
'error': self.SQL_ERROR,
'comment': self.SQL_COMMENT,
'table-missing-in-db': self.SQL_TABLE_MISSING_IN_DB,
'table-missing-in-model': self.SQL_TABLE_MISSING_IN_MODEL,
'field-missing-in-db': self.SQL_FIELD_MISSING_IN_DB,
'field-missing-in-model': self.SQL_FIELD_MISSING_IN_MODEL,
'fkey-missing-in-db': self.SQL_FKEY_MISSING_IN_DB,
'fkey-missing-in-model': self.SQL_FIELD_MISSING_IN_MODEL,
'index-missing-in-db': self.SQL_INDEX_MISSING_IN_DB,
'index-missing-in-model': self.SQL_INDEX_MISSING_IN_MODEL,
'unique-missing-in-db': self.SQL_UNIQUE_MISSING_IN_DB,
'unique-missing-in-model': self.SQL_UNIQUE_MISSING_IN_MODEL,
'field-type-differ': self.SQL_FIELD_TYPE_DIFFER,
'field-parameter-differ': self.SQL_FIELD_PARAMETER_DIFFER,
'notnull-differ': self.SQL_NOTNULL_DIFFER,
}
if self.can_detect_notnull_differ:
self.load_null()
if self.can_detect_unsigned_differ:
self.load_unsigned()
def __init__(self, app_models, options):
self.has_differences = None
self.app_models = app_models
self.options = options
self.dense = options.get('dense_output', False)
try:
self.introspection = connection.introspection
except AttributeError:
from django.db import get_introspection_module
self.introspection = get_introspection_module()
self.cursor = connection.cursor()
self.django_tables = self.get_django_tables(options.get('only_existing', True))
# TODO: We are losing information about tables which are views here
self.db_tables = [table_info.name for table_info in self.introspection.get_table_list(self.cursor)]
self.differences = []
self.unknown_db_fields = {}
self.new_db_fields = set()
self.null = {}
self.unsigned = set()
self.DIFF_SQL = {
'error': self.SQL_ERROR,
'comment': self.SQL_COMMENT,
'table-missing-in-db': self.SQL_TABLE_MISSING_IN_DB,
'table-missing-in-model': self.SQL_TABLE_MISSING_IN_MODEL,
'field-missing-in-db': self.SQL_FIELD_MISSING_IN_DB,
'field-missing-in-model': self.SQL_FIELD_MISSING_IN_MODEL,
'fkey-missing-in-db': self.SQL_FKEY_MISSING_IN_DB,
'fkey-missing-in-model': self.SQL_FIELD_MISSING_IN_MODEL,
'index-missing-in-db': self.SQL_INDEX_MISSING_IN_DB,
'index-missing-in-model': self.SQL_INDEX_MISSING_IN_MODEL,
'unique-missing-in-db': self.SQL_UNIQUE_MISSING_IN_DB,
'unique-missing-in-model': self.SQL_UNIQUE_MISSING_IN_MODEL,
'field-type-differ': self.SQL_FIELD_TYPE_DIFFER,
'field-parameter-differ': self.SQL_FIELD_PARAMETER_DIFFER,
'notnull-differ': self.SQL_NOTNULL_DIFFER,
}
if self.can_detect_notnull_differ:
self.load_null()
if self.can_detect_unsigned_differ:
self.load_unsigned()
def __init__(self, app_models, options):
self.has_differences = None
self.app_models = app_models
self.options = options
self.dense = options.get("dense_output", False)
try:
self.introspection = connection.introspection
except AttributeError:
from django.db import get_introspection_module
self.introspection = get_introspection_module()
self.cursor = connection.cursor()
self.django_tables = self.get_django_tables(options.get("only_existing", True))
self.db_tables = self.introspection.get_table_list(self.cursor)
self.differences = []
self.unknown_db_fields = {}
self.new_db_fields = set()
self.null = {}
self.unsigned = set()
self.DIFF_SQL = {
"error": self.SQL_ERROR,
"comment": self.SQL_COMMENT,
"table-missing-in-db": self.SQL_TABLE_MISSING_IN_DB,
"table-missing-in-model": self.SQL_TABLE_MISSING_IN_MODEL,
"field-missing-in-db": self.SQL_FIELD_MISSING_IN_DB,
"field-missing-in-model": self.SQL_FIELD_MISSING_IN_MODEL,
"fkey-missing-in-db": self.SQL_FKEY_MISSING_IN_DB,
"fkey-missing-in-model": self.SQL_FIELD_MISSING_IN_MODEL,
"index-missing-in-db": self.SQL_INDEX_MISSING_IN_DB,
"index-missing-in-model": self.SQL_INDEX_MISSING_IN_MODEL,
"unique-missing-in-db": self.SQL_UNIQUE_MISSING_IN_DB,
"unique-missing-in-model": self.SQL_UNIQUE_MISSING_IN_MODEL,
"field-type-differ": self.SQL_FIELD_TYPE_DIFFER,
"field-parameter-differ": self.SQL_FIELD_PARAMETER_DIFFER,
"notnull-differ": self.SQL_NOTNULL_DIFFER,
}
if self.can_detect_notnull_differ:
self.load_null()
if self.can_detect_unsigned_differ:
self.load_unsigned()
def get_sql_evolution_check_for_changed_model_name(klass, style):
from django.db import get_introspection_module, connection
ops, introspection = get_operations_and_introspection_classes(style)
cursor = connection.cursor()
introspection = get_introspection_module()
# ops = get_ops_class(connection)
table_list = introspection.get_table_list(cursor)
if klass._meta.db_table in table_list:
return [], None
aka_db_tables = set()
if klass._meta.aka:
for x in klass._meta.aka:
aka_db_tables.add( "%s_%s" % (klass._meta.app_label, x.lower()) )
matches = list(aka_db_tables & set(table_list))
if len(matches)==1:
return ops.get_change_table_name_sql( klass._meta.db_table, matches[0]), matches[0]
else:
return [], None
def handle_inspection(self):
from django.db import connection, get_introspection_module
import keyword
introspection_module = get_introspection_module()
table2model = lambda table_name: table_name.title().replace('_', '')
cursor = connection.cursor()
yield "# This is an auto-generated Django model module."
yield "# You'll have to do the following manually to clean this up:"
yield "# * Rearrange models' order"
yield "# * Make sure each model has one field with primary_key=True"
yield "# Feel free to rename the models, but don't rename db_table values or field names."
yield "#"
yield "# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'"
yield "# into your database."
yield ''
yield 'from django.db import models'
yield ''
for table_name in introspection_module.get_table_list(cursor):
yield 'class %s(models.Model):' % table2model(table_name)
try:
relations = introspection_module.get_relations(cursor, table_name)
except NotImplementedError:
relations = {}
try:
indexes = introspection_module.get_indexes(cursor, table_name)
except NotImplementedError:
indexes = {}
for i, row in enumerate(introspection_module.get_table_description(cursor, table_name)):
att_name = row[0].lower()
comment_notes = [] # Holds Field notes, to be displayed in a Python comment.
extra_params = {} # Holds Field parameters such as 'db_column'.
if ' ' in att_name:
extra_params['db_column'] = att_name
att_name = att_name.replace(' ', '')
comment_notes.append('Field renamed to remove spaces.')
if keyword.iskeyword(att_name):
extra_params['db_column'] = att_name
att_name += '_field'
comment_notes.append('Field renamed because it was a Python reserved word.')
if i in relations:
rel_to = relations[i][1] == table_name and "'self'" or table2model(relations[i][1])
field_type = 'ForeignKey(%s' % rel_to
if att_name.endswith('_id'):
att_name = att_name[:-3]
else:
extra_params['db_column'] = att_name
else:
try:
field_type = introspection_module.DATA_TYPES_REVERSE[row[1]]
except KeyError:
field_type = 'TextField'
comment_notes.append('This field type is a guess.')
# This is a hook for DATA_TYPES_REVERSE to return a tuple of
# (field_type, extra_params_dict).
if type(field_type) is tuple:
field_type, new_params = field_type
extra_params.update(new_params)
# Add max_length for all CharFields.
if field_type == 'CharField' and row[3]:
extra_params['max_length'] = row[3]
if field_type == 'DecimalField':
extra_params['max_digits'] = row[4]
extra_params['decimal_places'] = row[5]
# Add primary_key and unique, if necessary.
column_name = extra_params.get('db_column', att_name)
if column_name in indexes:
if indexes[column_name]['primary_key']:
extra_params['primary_key'] = True
elif indexes[column_name]['unique']:
extra_params['unique'] = True
field_type += '('
# Don't output 'id = meta.AutoField(primary_key=True)', because
# that's assumed if it doesn't exist.
if att_name == 'id' and field_type == 'AutoField(' and extra_params == {'primary_key': True}:
continue
# Add 'null' and 'blank', if the 'null_ok' flag was present in the
# table description.
if row[6]: # If it's NULL...
extra_params['blank'] = True
if not field_type in ('TextField(', 'CharField('):
extra_params['null'] = True
field_desc = '%s = models.%s' % (att_name, field_type)
if extra_params:
if not field_desc.endswith('('):
field_desc += ', '
field_desc += ', '.join(['%s=%r' % (k, v) for k, v in list(extra_params.items())])
field_desc += ')'
if comment_notes:
field_desc += ' # ' + ' '.join(comment_notes)
yield ' %s' % field_desc
yield ' class Meta:'
yield ' db_table = %r' % table_name
yield ''
def sql_delete(app, style):
"Returns a list of the DROP TABLE SQL statements for the given app."
from django.db import connection, models, get_introspection_module
from django.db.backends.util import truncate_name
from django.contrib.contenttypes import generic
introspection = get_introspection_module()
# This should work even if a connection isn't available
try:
cursor = connection.cursor()
except:
cursor = None
# Figure out which tables already exist
if cursor:
table_names = introspection.get_table_list(cursor)
else:
table_names = []
if connection.features.uses_case_insensitive_names:
table_name_converter = lambda x: x.upper()
else:
table_name_converter = lambda x: x
output = []
qn = connection.ops.quote_name
# Output DROP TABLE statements for standard application tables.
to_delete = set()
references_to_delete = {}
app_models = models.get_models(app)
for model in app_models:
if cursor and table_name_converter(model._meta.db_table) in table_names:
# The table exists, so it needs to be dropped
opts = model._meta
for f in opts.fields:
if f.rel and f.rel.to not in to_delete:
references_to_delete.setdefault(f.rel.to, []).append( (model, f) )
to_delete.add(model)
for model in app_models:
if cursor and table_name_converter(model._meta.db_table) in table_names:
# Drop the table now
output.append('%s %s;' % (style.SQL_KEYWORD('DROP TABLE'),
style.SQL_TABLE(qn(model._meta.db_table))))
if connection.features.supports_constraints and model in references_to_delete:
for rel_class, f in references_to_delete[model]:
table = rel_class._meta.db_table
col = f.column
r_table = model._meta.db_table
r_col = model._meta.get_field(f.rel.field_name).column
r_name = '%s_refs_%s_%x' % (col, r_col, abs(hash((table, r_table))))
output.append('%s %s %s %s;' % \
(style.SQL_KEYWORD('ALTER TABLE'),
style.SQL_TABLE(qn(table)),
style.SQL_KEYWORD(connection.ops.drop_foreignkey_sql()),
style.SQL_FIELD(truncate_name(r_name, connection.ops.max_name_length()))))
del references_to_delete[model]
if model._meta.has_auto_field:
ds = connection.ops.drop_sequence_sql(model._meta.db_table)
if ds:
output.append(ds)
# Output DROP TABLE statements for many-to-many tables.
for model in app_models:
opts = model._meta
for f in opts.many_to_many:
if isinstance(f.rel, generic.GenericRel):
continue
if cursor and table_name_converter(f.m2m_db_table()) in table_names:
output.append("%s %s;" % (style.SQL_KEYWORD('DROP TABLE'),
style.SQL_TABLE(qn(f.m2m_db_table()))))
ds = connection.ops.drop_sequence_sql("%s_%s" % (model._meta.db_table, f.column))
if ds:
output.append(ds)
app_label = app_models[0]._meta.app_label
# Close database connection explicitly, in case this output is being piped
# directly into a database client, to avoid locking issues.
if cursor:
cursor.close()
connection.close()
return output[::-1] # Reverse it, to deal with table dependencies.
def handle_inspection(self):
from django.db import connection, get_introspection_module
import keyword
introspection_module = get_introspection_module()
table2model = lambda table_name: table_name.title().replace('_', '')
cursor = connection.cursor()
yield "# This is an auto-generated Django model module."
yield "# You'll have to do the following manually to clean this up:"
yield "# * Rearrange models' order"
yield "# * Make sure each model has one field with primary_key=True"
yield "# Feel free to rename the models, but don't rename db_table values or field names."
yield "#"
yield "# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'"
yield "# into your database."
yield ''
yield 'from django.db import models'
yield ''
for table_name in introspection_module.get_table_list(cursor):
yield 'class %s(models.Model):' % table2model(table_name)
try:
relations = introspection_module.get_relations(
cursor, table_name)
except NotImplementedError:
relations = {}
try:
indexes = introspection_module.get_indexes(cursor, table_name)
except NotImplementedError:
indexes = {}
for i, row in enumerate(
introspection_module.get_table_description(
cursor, table_name)):
att_name = row[0].lower()
comment_notes = [
] # Holds Field notes, to be displayed in a Python comment.
extra_params = {
} # Holds Field parameters such as 'db_column'.
if ' ' in att_name:
extra_params['db_column'] = att_name
att_name = att_name.replace(' ', '')
comment_notes.append('Field renamed to remove spaces.')
if keyword.iskeyword(att_name):
extra_params['db_column'] = att_name
att_name += '_field'
comment_notes.append(
'Field renamed because it was a Python reserved word.')
if i in relations:
rel_to = relations[i][
1] == table_name and "'self'" or table2model(
relations[i][1])
field_type = 'ForeignKey(%s' % rel_to
if att_name.endswith('_id'):
att_name = att_name[:-3]
else:
extra_params['db_column'] = att_name
else:
try:
field_type = introspection_module.DATA_TYPES_REVERSE[
row[1]]
except KeyError:
field_type = 'TextField'
comment_notes.append('This field type is a guess.')
# This is a hook for DATA_TYPES_REVERSE to return a tuple of
# (field_type, extra_params_dict).
if type(field_type) is tuple:
field_type, new_params = field_type
extra_params.update(new_params)
# Add max_length for all CharFields.
if field_type == 'CharField' and row[3]:
extra_params['max_length'] = row[3]
if field_type == 'DecimalField':
extra_params['max_digits'] = row[4]
extra_params['decimal_places'] = row[5]
# Add primary_key and unique, if necessary.
column_name = extra_params.get('db_column', att_name)
if column_name in indexes:
if indexes[column_name]['primary_key']:
extra_params['primary_key'] = True
elif indexes[column_name]['unique']:
extra_params['unique'] = True
field_type += '('
# Don't output 'id = meta.AutoField(primary_key=True)', because
# that's assumed if it doesn't exist.
if att_name == 'id' and field_type == 'AutoField(' and extra_params == {
'primary_key': True
}:
continue
# Add 'null' and 'blank', if the 'null_ok' flag was present in the
# table description.
if row[6]: # If it's NULL...
extra_params['blank'] = True
if not field_type in ('TextField(', 'CharField('):
extra_params['null'] = True
field_desc = '%s = models.%s' % (att_name, field_type)
if extra_params:
if not field_desc.endswith('('):
field_desc += ', '
field_desc += ', '.join(
['%s=%r' % (k, v) for k, v in extra_params.items()])
field_desc += ')'
if comment_notes:
field_desc += ' # ' + ' '.join(comment_notes)
yield ' %s' % field_desc
yield ' class Meta:'
yield ' db_table = %r' % table_name
yield ''
def table_list():
"Returns a list of all table names that exist in the database."
from django.db import connection, get_introspection_module
cursor = connection.cursor()
return get_introspection_module().get_table_list(cursor)
def get_introspected_evolution_options(app, style):
ops, introspection = get_operations_and_introspection_classes(style)
from django.db import models, get_introspection_module, connection
cursor = connection.cursor()
app_name = app.__name__.split('.')[-2]
final_output = []
table_list = get_introspection_module().get_table_list(cursor)
seen_models = management.installed_models(table_list)
created_models = set()
final_output = []
seen_tables = set()
model_list = models.get_models(app)
for model in model_list:
# Create the model's database table, if it doesn't already exist.
aka_db_tables = set()
if model._meta.aka:
for x in model._meta.aka:
aka_db_tables.add( "%s_%s" % (model._meta.app_label, x.lower()) )
if model._meta.db_table in table_list or len(aka_db_tables & set(table_list))>0:
continue #renamed
sql, references = fixed_sql_model_create(model, seen_models, style)
final_output.extend(sql)
seen_models.add(model)
created_models.add(model)
table_list.append(model._meta.db_table)
seen_tables.add(model._meta.db_table)
# get the existing models, minus the models we've just created
app_models = models.get_models(app)
for model in created_models:
if model in app_models:
app_models.remove(model)
for model in app_models:
if model._meta.db_table:
seen_tables.add(model._meta.db_table)
output, old_table_name = get_sql_evolution_check_for_changed_model_name(model, style)
if old_table_name: seen_tables.add(old_table_name)
final_output.extend(output)
rebuild = False
output = get_sql_evolution_check_for_changed_field_flags(model, old_table_name, style)
if output and settings.DATABASE_ENGINE == 'sqlite3': rebuild = True
final_output.extend(output)
output = get_sql_evolution_check_for_changed_field_name(model, old_table_name, style)
final_output.extend(output)
if output and settings.DATABASE_ENGINE == 'sqlite3': rebuild = True
output = get_sql_evolution_check_for_new_fields(model, old_table_name, style)
if output and settings.DATABASE_ENGINE == 'sqlite3': rebuild = True
final_output.extend(output)
output = get_sql_evolution_check_for_dead_fields(model, old_table_name, style)
if output and settings.DATABASE_ENGINE == 'sqlite3': rebuild = True
final_output.extend(output)
if rebuild:
output = get_sql_evolution_rebuild_table(model, old_table_name, style)
final_output.extend(output)
for model in app_models + list(created_models):
for f in model._meta.many_to_many:
#creating many_to_many table
if not f.m2m_db_table() in get_introspection_module().get_table_list(cursor):
final_output.extend( _get_many_to_many_sql_for_field(model, f, style) )
seen_tables.add(f.m2m_db_table())
output = get_sql_evolution_check_for_dead_models(table_list, seen_tables, app_name, app_models, style)
final_output.extend(output)
return final_output
def handle_diff(self, app, **options):
from django.db import models, connection
from django.core.management import sql as _sql
app_name = app.__name__.split('.')[-2]
try:
django_tables = connection.introspection.django_table_names(only_existing=options.get('only_existing', True))
except AttributeError:
# backwards compatibility for before introspection refactoring (r8296)
try:
django_tables = _sql.django_table_names(only_existing=options.get('only_existing', True))
except AttributeError:
# backwards compatibility for before svn r7568
django_tables = _sql.django_table_list(only_existing=options.get('only_existing', True))
django_tables = [django_table for django_table in django_tables if django_table.startswith(app_name)]
app_models = models.get_models(app)
if not app_models:
return
try:
from django.db import get_introspection_module
introspection_module = get_introspection_module()
except ImportError:
introspection_module = connection.introspection
cursor = connection.cursor()
model_diffs = []
for app_model in app_models:
_constraints = None
_meta = app_model._meta
table_name = _meta.db_table
table_indexes = introspection_module.get_indexes(cursor, table_name)
fieldmap = dict([(field.get_attname(), field) for field in _meta.fields])
if _meta.order_with_respect_to:
fieldmap['_order'] = ORDERING_FIELD
try:
table_description = introspection_module.get_table_description(cursor, table_name)
except Exception, e:
model_diffs.append((app_model.__name__, [str(e).strip()]))
transaction.rollback() # reset transaction
continue
diffs = []
for i, row in enumerate(table_description):
att_name = row[0].lower()
try:
db_field_reverse_type = introspection_module.data_types_reverse[row[1]]
except AttributeError:
# backwards compatibility for before introspection refactoring (r8296)
db_field_reverse_type = introspection_module.DATA_TYPES_REVERSE.get(row[1])
kwargs = {}
if isinstance(db_field_reverse_type, tuple):
kwargs.update(db_field_reverse_type[1])
db_field_reverse_type = db_field_reverse_type[0]
if db_field_reverse_type == "CharField" and row[3]:
kwargs['max_length'] = row[3]
if db_field_reverse_type == "DecimalField":
kwargs['max_digits'] = row[4]
kwargs['decimal_places'] = row[5]
if row[6]:
kwargs['blank'] = True
if not db_field_reverse_type in ('TextField', 'CharField'):
kwargs['null'] = True
if fieldmap.has_key(att_name):
field = fieldmap.pop(att_name)
# check type
def clean(s):
s = s.split(" ")[0]
s = s.split("(")[0]
return s
def cmp_or_serialcmp(x, y):
result = x==y
if result:
return result
is_serial = lambda x,y: x.startswith("serial") and y.startswith("integer")
strip_serial = lambda x: x.lstrip("serial").lstrip("integer")
serial_logic = is_serial(x, y) or is_serial(y, x)
if result==False and serial_logic:
# use alternate serial logic
result = strip_serial(x)==strip_serial(y)
return result
db_field_type = getattr(models, db_field_reverse_type)(**kwargs).db_type()
model_type = field.db_type()
# remove mysql's auto_increment keyword
if self.is_mysql and model_type.endswith("AUTO_INCREMENT"):
model_type = model_type.rsplit(' ', 1)[0].strip()
# check if we can for constraints (only enabled on postgresql atm)
if self.is_pgsql:
if _constraints==None:
sql = """
SELECT
pg_constraint.conname, pg_get_constraintdef(pg_constraint.oid)
FROM
pg_constraint, pg_attribute
WHERE
pg_constraint.conrelid = pg_attribute.attrelid
AND pg_attribute.attnum = any(pg_constraint.conkey)
AND pg_constraint.conname ~ %s"""
cursor.execute(sql, [table_name])
_constraints = [r for r in cursor.fetchall() if r[0].endswith("_check")]
for r_name, r_check in _constraints:
if table_name+"_"+att_name==r_name.rsplit("_check")[0]:
r_check = r_check.replace("((", "(").replace("))", ")")
pos = r_check.find("(")
r_check = "%s\"%s" % (r_check[:pos+1], r_check[pos+1:])
pos = pos+r_check[pos:].find(" ")
r_check = "%s\" %s" % (r_check[:pos], r_check[pos+1:])
db_field_type += " "+r_check
else:
# remove constraints
model_type = model_type.split("CHECK")[0].strip()
c_db_field_type = clean(db_field_type)
c_model_type = clean(model_type)
if self.is_sqlite and (c_db_field_type=="varchar" and c_model_type=="char"):
c_db_field_type = "char"
db_field_type = db_field_type.lstrip("var")
if not cmp_or_serialcmp(c_model_type, c_db_field_type):
diffs.append({
'text' : "field '%s' not of same type: db=%s, model=%s" % (att_name, c_db_field_type, c_model_type),
'type' : 'type',
'data' : (table_name, att_name, c_db_field_type, c_model_type)
})
continue
if not cmp_or_serialcmp(db_field_type, model_type):
diffs.append({
'text' : "field '%s' parameters differ: db=%s, model=%s" % (att_name, db_field_type, model_type),
'type' : 'param',
'data' : (table_name, att_name, db_field_type, model_type)
})
continue
else:
diffs.append({
'text' : "field '%s' missing in model: %s" % (att_name, model_type),
'type' : 'missing-in-model',
'data' : (table_name, att_name, db_field_type, model_type)
})
for field in _meta.fields:
if field.db_index:
if not field.attname in table_indexes and not field.unique:
diffs.append({
'text' : "field '%s' INDEX defined in model missing in database" % (field.attname),
})
if fieldmap:
for att_name, field in fieldmap.items():
diffs.append({
'text' : "field '%s' missing in database: %s" % (att_name, field.db_type()),
'type' : 'missing-in-db',
'data' : (table_name, att_name, field.db_type())
})
if diffs:
model_diffs.append((app_model.__name__, diffs))
def table_names():
"Returns a list of all table names that exist in the database."
from django.db import connection, get_introspection_module
cursor = connection.cursor()
return set(get_introspection_module().get_table_list(cursor))
def inspectdb(tableOrder=[]):
"Generator that introspects the tables in the given database name and returns a Django model, one line at a time."
from django.db import connection, get_introspection_module
import keyword
introspection_module = get_introspection_module()
#table2model = lambda table_name: table_name.title().replace('_', '')
table2model = lambda table_name: table_name
cursor = connection.cursor()
yield "# This is an auto-generated customized Django model module for the IRAM-30m Archive"
yield "# If you provide an SQL script, the models' order should be OK"
yield "# In other case, please rearrange models' order!"
yield "#"
yield "# Also Many to Many tables ('*_has_*' or '*_usedin_*') are ignored and replaced with ManyToManyField. Please check it!"
yield "#"
yield "# You'll have to do the following manually to clean this up:"
yield "# * Replace any reference to model User to use django.user (e.g %s/DjangoUsers/User/g)"
yield "# * Fix CharField max_length: 60->20 ; 135 -> 45 ; 765 -> 255 "
yield "# * Replace IntegerField with BooleanField when needed "
yield "# Feel free to rename the models, but don't rename db_table values or field names."
yield "#"
yield "# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'"
yield "# into your database."
yield ''
yield 'from django.db import models'
yield 'from django.contrib.auth.models import User'
yield ''
tablesList = sortListByList( introspection_module.get_table_list(cursor), tableOrder )
pattern = re.compile("_(has|usedin)_", re.IGNORECASE)
#remove the many2many table
for table_name in filter( lambda i: not pattern.search(i), tablesList ):
yield 'class %s(models.Model):' % table2model(table_name)
try:
relations = introspection_module.get_relations(cursor, table_name)
except NotImplementedError:
relations = {}
try:
indexes = introspection_module.get_indexes(cursor, table_name)
except NotImplementedError:
indexes = {}
for i, row in enumerate(introspection_module.get_table_description(cursor, table_name)):
att_name = row[0].lower()
comment_notes = [] # Holds Field notes, to be displayed in a Python comment.
extra_params = {} # Holds Field parameters such as 'db_column'.
if ' ' in att_name:
extra_params['db_column'] = att_name
att_name = att_name.replace(' ', '')
comment_notes.append('Field renamed to remove spaces.')
if keyword.iskeyword(att_name):
extra_params['db_column'] = att_name
att_name += '_field'
comment_notes.append('Field renamed because it was a Python reserved word.')
if i in relations:
rel_to = relations[i][1] == table_name and "'self'" or table2model(relations[i][1])
field_type = 'ForeignKey(%s' % rel_to
if att_name.endswith('_id'):
att_name = att_name[:-3]
else:
extra_params['db_column'] = att_name
else:
try:
field_type = introspection_module.DATA_TYPES_REVERSE[row[1]]
except KeyError:
field_type = 'TextField'
comment_notes.append('This field type is a guess.')
# This is a hook for DATA_TYPES_REVERSE to return a tuple of
# (field_type, extra_params_dict).
if type(field_type) is tuple:
field_type, new_params = field_type
extra_params.update(new_params)
# Add maxlength for all CharFields.
if field_type == 'CharField' and row[3]:
extra_params['max_length'] = row[3]
if field_type == 'DecimalField':
extra_params['max_digits'] = row[4]
extra_params['decimal_places'] = row[5]
# Add primary_key and unique, if necessary.
column_name = extra_params.get('db_column', att_name)
if column_name in indexes:
if indexes[column_name]['primary_key']:
extra_params['primary_key'] = True
elif indexes[column_name]['unique']:
extra_params['unique'] = True
field_type += '('
# Don't output 'id = meta.AutoField(primary_key=True)', because
# that's assumed if it doesn't exist.
if att_name == 'id' and field_type == 'AutoField(' and extra_params == {'primary_key': True}:
continue
if att_name == 'id' and field_type == 'IntegerField(' and extra_params == {'primary_key': True}:
continue
# Add 'null' and 'blank', if the 'null_ok' flag was present in the
# table description.
if row[6]: # If it's NULL...
extra_params['blank'] = True
if not field_type in ('TextField(', 'CharField('):
extra_params['null'] = True
field_desc = '%s = models.%s' % (att_name, field_type)
if extra_params:
if not field_desc.endswith('('):
field_desc += ', '
field_desc += ', '.join(['%s=%r' % (k, v) for k, v in extra_params.items()])
field_desc += ')'
if comment_notes:
field_desc += ' # ' + ' '.join(comment_notes)
yield ' %s' % field_desc
pattern = re.compile( table_name + "_has_(.+)|(.+)_usedin_"+table_name ,re.IGNORECASE )
# filter gets all the table with many2many names
tablesRelatedTo = filter(pattern.search, tablesList)
if tablesRelatedTo:
# and map transforms that list to a list with matches
matchesRelatedTo = map(pattern.findall, tablesRelatedTo)
for table,i in zip(matchesRelatedTo, range(len(matchesRelatedTo))):
comment = "Automatically generated!"
yield ' %s = models.ManyToManyField(db_table = "%s") # %s' % (table[0][0].lower() or table[0][1].lower(), tablesRelatedTo[i], comment)
yield ' class Meta:'
yield ' db_table = %r' % table_name
yield ''
def sql_delete(app, style):
"Returns a list of the DROP TABLE SQL statements for the given app."
from django.db import connection, models, get_introspection_module
from django.db.backends.util import truncate_name
from django.contrib.contenttypes import generic
introspection = get_introspection_module()
# This should work even if a connection isn't available
try:
cursor = connection.cursor()
except:
cursor = None
# Figure out which tables already exist
if cursor:
table_names = introspection.get_table_list(cursor)
else:
table_names = []
if connection.features.uses_case_insensitive_names:
table_name_converter = lambda x: x.upper()
else:
table_name_converter = lambda x: x
output = []
qn = connection.ops.quote_name
# Output DROP TABLE statements for standard application tables.
to_delete = set()
references_to_delete = {}
app_models = models.get_models(app)
for model in app_models:
if cursor and table_name_converter(
model._meta.db_table) in table_names:
# The table exists, so it needs to be dropped
opts = model._meta
for f in opts.local_fields:
if f.rel and f.rel.to not in to_delete:
references_to_delete.setdefault(f.rel.to, []).append(
(model, f))
to_delete.add(model)
for model in app_models:
if cursor and table_name_converter(
model._meta.db_table) in table_names:
# Drop the table now
output.append('%s %s;' %
(style.SQL_KEYWORD('DROP TABLE'),
style.SQL_TABLE(qn(model._meta.db_table))))
if connection.features.supports_constraints and model in references_to_delete:
for rel_class, f in references_to_delete[model]:
table = rel_class._meta.db_table
col = f.column
r_table = model._meta.db_table
r_col = model._meta.get_field(f.rel.field_name).column
r_name = '%s_refs_%s_%x' % (col, r_col,
abs(hash((table, r_table))))
output.append('%s %s %s %s;' % \
(style.SQL_KEYWORD('ALTER TABLE'),
style.SQL_TABLE(qn(table)),
style.SQL_KEYWORD(connection.ops.drop_foreignkey_sql()),
style.SQL_FIELD(truncate_name(r_name, connection.ops.max_name_length()))))
del references_to_delete[model]
if model._meta.has_auto_field:
ds = connection.ops.drop_sequence_sql(model._meta.db_table)
if ds:
output.append(ds)
# Output DROP TABLE statements for many-to-many tables.
for model in app_models:
opts = model._meta
for f in opts.local_many_to_many:
if isinstance(f.rel, generic.GenericRel):
continue
if cursor and table_name_converter(
f.m2m_db_table()) in table_names:
output.append("%s %s;" %
(style.SQL_KEYWORD('DROP TABLE'),
style.SQL_TABLE(qn(f.m2m_db_table()))))
ds = connection.ops.drop_sequence_sql(
"%s_%s" % (model._meta.db_table, f.column))
if ds:
output.append(ds)
app_label = app_models[0]._meta.app_label
# Close database connection explicitly, in case this output is being piped
# directly into a database client, to avoid locking issues.
if cursor:
cursor.close()
connection.close()
return output[::-1] # Reverse it, to deal with table dependencies.
def handle_diff(self, app, **options):
from django.db import models, connection, get_introspection_module
from django.core.management import sql as _sql
app_name = app.__name__.split('.')[-2]
try:
django_tables = _sql.django_table_names(only_existing=options.get('only_existing', True))
except AttributeError:
# backwards compatibility for before svn r7568
django_tables = _sql.django_table_list(only_existing=options.get('only_existing', True))
django_tables = [django_table for django_table in django_tables if django_table.startswith(app_name)]
app_models = models.get_models(app)
if not app_models:
return
introspection_module = get_introspection_module()
cursor = connection.cursor()
model_diffs = []
for app_model in app_models:
_constraints = None
_meta = app_model._meta
table_name = _meta.db_table
table_indexes = introspection_module.get_indexes(cursor, table_name)
fieldmap = dict([(field.get_attname(), field) for field in _meta.fields])
try:
table_description = introspection_module.get_table_description(cursor, table_name)
except Exception, e:
model_diffs.append((app_model.__name__, [str(e).strip()]))
transaction.rollback() # reset transaction
continue
diffs = []
for i, row in enumerate(table_description):
att_name = row[0].lower()
db_field_reverse_type = introspection_module.DATA_TYPES_REVERSE.get(row[1])
kwargs = {}
if row[3]:
kwargs['max_length'] = row[3]
if row[4]:
kwargs['max_digits'] = row[4]
if row[5]:
kwargs['decimal_places'] = row[5]
if row[6]:
kwargs['blank'] = True
if not db_field_reverse_type in ('TextField', 'CharField'):
extra_params['null'] = True
if fieldmap.has_key(att_name):
field = fieldmap.pop(att_name)
# check type
def clean(s):
s = s.split(" ")[0]
s = s.split("(")[0]
return s
def cmp_or_serialcmp(x, y):
result = x==y
if result:
return result
is_serial = lambda x,y: x.startswith("serial") and y.startswith("integer")
strip_serial = lambda x: x.lstrip("serial").lstrip("integer")
serial_logic = is_serial(x, y) or is_serial(y, x)
if result==False and serial_logic:
# use alternate serial logic
result = strip_serial(x)==strip_serial(y)
return result
db_field_type = getattr(models, db_field_reverse_type)(**kwargs).db_type()
model_type = field.db_type()
# check if we can for constraints (only enabled on postgresql atm)
if self.is_pgsql:
if _constraints==None:
sql = """
SELECT
pg_constraint.conname, pg_get_constraintdef(pg_constraint.oid)
FROM
pg_constraint, pg_attribute
WHERE
pg_constraint.conrelid = pg_attribute.attrelid
AND pg_attribute.attnum = any(pg_constraint.conkey)
AND pg_constraint.conname ~ %s"""
cursor.execute(sql, [table_name])
_constraints = [r for r in cursor.fetchall() if r[0].endswith("_check")]
for r_name, r_check in _constraints:
if table_name+"_"+att_name==r_name.rsplit("_check")[0]:
r_check = r_check.replace("((", "(").replace("))", ")")
pos = r_check.find("(")
r_check = "%s\"%s" % (r_check[:pos+1], r_check[pos+1:])
pos = pos+r_check[pos:].find(" ")
r_check = "%s\" %s" % (r_check[:pos], r_check[pos+1:])
db_field_type += " "+r_check
else:
# remove constraints
model_type = model_type.split("CHECK")[0].strip()
c_db_field_type = clean(db_field_type)
c_model_type = clean(model_type)
if not cmp_or_serialcmp(c_model_type, c_db_field_type):
diffs.append("field '%s' not of same type: db=%s, model=%s" % (att_name, c_db_field_type, c_model_type))
continue
if not cmp_or_serialcmp(db_field_type, model_type):
diffs.append("field '%s' parameters differ: db=%s, model=%s" % (att_name, db_field_type, model_type))
continue
else:
diffs.append("field '%s' missing in model field" % att_name)
for field in _meta.fields:
if field.db_index:
if not field.attname in table_indexes and not field.unique:
diffs.append("field '%s' INDEX defined in model missing in database" % (field.attname))
if fieldmap:
for att_name, field in fieldmap.items():
diffs.append("field '%s' missing in database" % att_name)
if diffs:
model_diffs.append((app_model.__name__, diffs))