def Deserializer(object_list, **options):
"""
Deserialize simple Python objects back into Django ORM instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
db = options.pop('using', DEFAULT_DB_ALIAS)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__'):
return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
else:
return smart_unicode(field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
if hasattr(field_value, '__iter__'):
obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value)
value = getattr(obj, field.rel.field_name)
# If this is a natural foreign key to an object that
# has a FK/O2O as the foreign key, use the FK value
if field.rel.to._meta.pk.rel:
value = value.pk
else:
value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
data[field.attname] = value
else:
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def validate(cls, model):
"""
Does basic PermissionHandler option validation. Calls custom validation
classmethod in the end if it is provided in cls. The signature of the
custom validation classmethod should be: def validate(cls, model).
"""
# Before we can introspect models, they need to be fully loaded so that
# inter-relations are set up correctly. We force that here.
models.get_apps()
# validate model class
if not isinstance(model, ModelBase):
# this mean the model is not subclass of models.Model
raise ValidationError('The model "%s" must be a subclass of ``models.Model``.' % model.__class__.__name__)
# validate handler class
if not issubclass(cls, PermissionHandler):
raise ValidationError(
'The handler "%s" must be a subclass of '
"``fluidpermission.handlers.PermissionHandler``." % cls.__class__.__name__
)
# call custom validation classmethod
if hasattr(cls, "validate"):
cls.validate(model)
def validate(cls, model):
""" Validates if model is well configured """
# Before we can introspect models, they need to be fully loaded so that
# inter-relations are set up correctly. We force that here.
models.get_apps()
# Call each option validation
validate_fields(cls, model)
def Deserializer(object_list, **options):
"""Deserialize simple Python objects back into Model instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = python._get_model(d["model"])
data = {}
key = resolve_key(Model._meta.module_name, d["pk"])
if key.name():
data["key_name"] = key.name()
parent = None
if key.parent():
parent = FakeParent(key.parent())
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(
field_value, options.get("encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model.properties()[field_name]
if isinstance(field, db.Reference):
# Resolve foreign key references.
data[field.name] = resolve_key(Model._meta.module_name, field_value)
else:
# Handle converting strings to more specific formats.
if isinstance(field_value, basestring):
if isinstance(field, db.DateProperty):
field_value = datetime.datetime.strptime(
field_value, '%Y-%m-%d').date()
elif isinstance(field, db.TimeProperty):
field_value = parse_datetime_with_microseconds(field_value,
'%H:%M:%S').time()
elif isinstance(field, db.DateTimeProperty):
field_value = parse_datetime_with_microseconds(field_value,
'%Y-%m-%d %H:%M:%S')
# Handle pyyaml datetime.time deserialization - it returns a datetime
# instead of a time.
if (isinstance(field_value, datetime.datetime) and
isinstance(field, db.TimeProperty)):
field_value = field_value.time()
data[field.name] = field.validate(field_value)
# Create the new model instance with all it's data, but no parent.
object = Model(**data)
# Now add the parent into the hidden attribute, bypassing the type checks
# in the Model's __init__ routine.
object._parent = parent
# When the deserialized object is saved our replacement DeserializedObject
# class will set object._parent to force the real parent model to be loaded
# the first time it is referenced.
yield base.DeserializedObject(object, m2m_data)
def __init__(self, stream_or_string, **options):
models.get_apps()
super(Deserializer, self).__init__(stream_or_string, **options)
self.wb = xlrd.open_workbook(stream_or_string.name, formatting_info=True)
self.row_index=-1
self.sheet_index=-1
self.sheet=None
self.model=None
self.do_stop_iteration=False
def validate(cls, model):
"""
Does basic ModelAdmin option validation. Calls custom validation
classmethod in the end if it is provided in cls. The signature of the
custom validation classmethod should be: def validate(cls, model).
"""
# Before we can introspect models, they need to be fully loaded so that
# inter-relations are set up correctly. We force that here.
models.get_apps()
opts = model._meta
validate_base(cls, model)
# list_display
if hasattr(cls, 'list_display'):
check_isseq(cls, 'list_display', cls.list_display)
for idx, field in enumerate(cls.list_display):
if not callable(field):
if not hasattr(cls, field):
if not hasattr(model, field):
try:
opts.get_field(field)
except models.FieldDoesNotExist:
raise ImproperlyConfigured("%s.list_display[%d], %r is not a callable or an attribute of %r or found in the model %r."
% (cls.__name__, idx, field, cls.__name__, model._meta.object_name))
else:
# getattr(model, field) could be an X_RelatedObjectsDescriptor
f = fetch_attr(cls, model, opts, "list_display[%d]" % idx, field)
if isinstance(f, models.ManyToManyField):
raise ImproperlyConfigured("'%s.list_display[%d]', '%s' is a ManyToManyField which is not supported."
% (cls.__name__, idx, field))
# list_display_links
if hasattr(cls, 'list_display_links'):
check_isseq(cls, 'list_display_links', cls.list_display_links)
for idx, field in enumerate(cls.list_display_links):
fetch_attr(cls, model, opts, 'list_display_links[%d]' % idx, field)
if field not in cls.list_display:
raise ImproperlyConfigured("'%s.list_display_links[%d]'"
"refers to '%s' which is not defined in 'list_display'."
% (cls.__name__, idx, field))
# list_filter
if hasattr(cls, 'list_filter'):
check_isseq(cls, 'list_filter', cls.list_filter)
for idx, fpath in enumerate(cls.list_filter):
try:
get_fields_from_path(model, fpath)
except (NotRelationField, FieldDoesNotExist), e:
raise ImproperlyConfigured(
"'%s.list_filter[%d]' refers to '%s' which does not refer to a Field." % (
cls.__name__, idx, fpath
)
)
def Deserializer_with_debugging(original_function, object_list, **options):
from django.core.serializers.python import _get_model
from django.db import DEFAULT_DB_ALIAS
from django.utils.encoding import smart_unicode
from django.conf import settings
print "loading all: %s" % object_list
db = options.pop('using', DEFAULT_DB_ALIAS)
db_models.get_apps()
for d in object_list:
print "loading %s" % d
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, db_models.ManyToManyRel):
print " field = %s" % field
print " field.rel = %s" % field.rel
print " field.rel.to = %s" % field.rel.to
print " field.rel.to._default_manager = %s" % (
field.rel.to._default_manager)
print " field.rel.to.objects = %s" % (
field.rel.to.objects)
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__'):
return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
else:
return smart_unicode(field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
for i, pk in enumerate(field_value):
print " %s: converted %s to %s" % (field.name,
pk, m2m_data[field.name][i])
result = original_function(object_list, **options)
print " result = %s" % result
import traceback
traceback.print_stack()
return result
def Deserializer(object_list, **options):
"""
Deserialize simple Python objects back into Django ORM instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
uses_dictionary_lookup = False
data = {}
m2m_data = {}
if 'pk' in d:
data[Model._meta.pk.attname] = Model._meta.pk.to_python(d["pk"])
else:
uses_dictionary_lookup = True
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
m2m_convert = field.rel.to._meta.pk.to_python
rel_values = []
for rel_value in field_value:
try:
rel_values.append(m2m_convert(smart_unicode(rel_value)))
except ValidationError:
rel_values.append(rel_value)
m2m_data[field.name] = rel_values
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
try:
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
except ValidationError:
data[field.attname] = field_value
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield utils.DeserializedObject(Model, data, m2m_data)
def __init__(self, stream_or_string, **options): """ Init this serializer given a stream or a string """ self.options = options if isinstance(stream_or_string, basestring): self.stream = StringIO(stream_or_string) else: self.stream = stream_or_string # hack to make sure that the models have all been loaded before # deserialization starts (otherwise subclass calls to get_model() # and friends might fail...) models.get_apps()
def get_app(app=None, verbosity=0):
"""Uses django.db.models.get_app and fuzzywuzzy to get the models module for a django app
Retrieve an app module from an app name string, even if mispelled (uses fuzzywuzzy to find the best match)
To get a list of all the apps use `get_app(None)` or `get_app([]) or get_app(())`
To get a single random app use `get_app('')`
>>> get_app('call').__class__.__name__ == 'module'
True
>>> get_app('model').__name__ == 'miner.models'
True
>>> isinstance(get_app('whatever'), ModuleType)
True
>>> isinstance(get_app(''), ModuleType)
True
isinstance(get_app(), ModuleType)
False
isinstance(get_app(), list)
True
"""
# print 'get_app(', app
if not app:
if not isinstance(app, (type(None), list, tuple)):
if get_app.default:
return get_app(get_app.default)
else:
return models.get_apps()[-1]
else:
return [app_class.__package__ for app_class in models.get_apps() if app_class and app_class.__package__]
if isinstance(app, basestring) and app.strip().endswith('.models'):
return get_app(app[:-len('.models')])
if isinstance(app, ModuleType):
return app
# print 'type(' + repr(app) + ') = ' + repr(type(app))
try:
if verbosity > 1:
print 'Attempting django.models.get_app(%r)' % app
return models.get_app(app)
except:
if not app:
if verbosity:
print 'WARNING: app = %r, so returning None!' % app
return None
if verbosity > 2:
print 'Trying a fuzzy match on app = %r' % app
app_names = [app_class.__package__ for app_class in models.get_apps() if app_class and app_class.__package__]
fuzzy_app_name = fuzzy.extractOne(str(app), app_names)[0]
if verbosity:
print 'WARNING: Best fuzzy match for app name %r is %s' % (app, fuzzy_app_name)
return get_app(fuzzy_app_name)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
object_list = simplejson.load(stream)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
m2m_convert = field.rel.to._meta.pk.to_python
m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
# handle a dictionary which means to lookup the instance
# and get the primary key this way (works great on
# GFK values)
if isinstance(field_value, dict):
lookup_params = {}
for k, v in field_value.iteritems():
lookup_params[k.encode("ascii")] = v
field_value = field.rel.to._default_manager.get(**lookup_params).pk
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of YAML data.
********
All this is copied from the python base deserializer but for 2 lines
********
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
object_list = yaml.load(stream)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
m2m_convert = field.rel.to._meta.pk.to_python
m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
#These are those 2 lines ******
if isinstance(field_value, dict):
field_value = field.rel.to._default_manager.get(**field_value).pk
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def build_suite(self, test_labels, extra_tests=None, **kwargs):
suite = unittest.TestSuite()
if test_labels:
for label in test_labels:
if "." in label:
from django.test.simple import build_test
suite.addTest(build_test(label))
else:
sub_suite = self.find_tests_and_apps(label)
suite.addTest(sub_suite)
else:
from django.db.models import get_apps
for app in get_apps():
from django.test.simple import build_suite
suite.addTest(build_suite(app))
if extra_tests:
for test in extra_tests:
suite.addTest(test)
from django.test.simple import reorder_suite
from unittest import TestCase
return reorder_suite(suite, (TestCase,))
def handle(self, *app_labels, **options):
from django.db.models import get_app, get_apps
exclude = options.get('exclude', [])
excluded_apps = [get_app(app_label) for app_label in exclude]
if len(app_labels) == 0:
app_list = [app for app in get_apps() if app not in excluded_apps]
else:
app_list = [get_app(app_label) for app_label in app_labels]
for app in app_list:
name = app.__name__.replace('.models', '')
app_path = __import__(name, {}, {}, [name.split('.')[-1]]).__path__
try:
params = imp.find_module('generator', app_path)
except ImportError:
pass
else:
generator = imp.load_module(name + '.generator', *params)
print "Generating items for %s." % name.title()
start = time.time()
load_json(generator.generate())
now = time.time()
print "Generation completed in %s seconds" % (now - start)
def django_table_names(self, only_existing=False):
"""
Returns a list of all table names that have associated Django models and
are in INSTALLED_APPS.
If only_existing is True, the resulting list will only include the tables
that actually exist in the database.
"""
from django.db import models, router
tables = set()
for app in models.get_apps():
for model in models.get_models(app):
if not model._meta.managed:
continue
if not router.allow_migrate(self.connection.alias, model):
continue
tables.add(model._meta.db_table)
tables.update(f.m2m_db_table() for f in model._meta.local_many_to_many)
tables = list(tables)
if only_existing:
existing_tables = self.table_names()
tables = [
t
for t in tables
if self.table_name_converter(t) in existing_tables
]
return tables
def sequence_list(self):
"Returns a list of information about all DB sequences for all models in all apps."
from django.db import models, router
apps = models.get_apps()
sequence_list = []
for app in apps:
for model in models.get_models(app):
if not model._meta.managed:
continue
if model._meta.swapped:
continue
if not router.allow_migrate(self.connection.alias, model):
continue
for f in model._meta.local_fields:
if isinstance(f, models.AutoField):
sequence_list.append({'table': model._meta.db_table, 'column': f.column})
break # Only one AutoField is allowed per model, so don't bother continuing.
for f in model._meta.local_many_to_many:
# If this is an m2m using an intermediate table,
# we don't need to reset the sequence.
if f.rel.through is None:
sequence_list.append({'table': f.m2m_db_table(), 'column': None})
return sequence_list
def get_models(app_labels):
""" Gets a list of models for the given app labels, with some exceptions.
TODO: If a required model is referenced, it should also be included.
Or at least discovered with a get_or_create() call.
"""
from django.db.models import get_app, get_apps, get_model
from django.db.models import get_models as get_all_models
# These models are not to be output, e.g. because they can be generated automatically
# TODO: This should be "appname.modelname" string
EXCLUDED_MODELS = (ContentType, )
models = []
# If no app labels are given, return all
if not app_labels:
for app in get_apps():
models += [m for m in get_all_models(app) if m not in EXCLUDED_MODELS]
# Get all relevant apps
for app_label in app_labels:
# If a specific model is mentioned, get only that model
if "." in app_label:
app_label, model_name = app_label.split(".", 1)
models.append(get_model(app_label, model_name))
# Get all models for a given app
else:
models += [m for m in get_all_models(get_app(app_label)) if m not in EXCLUDED_MODELS]
return models
def handle(self, *args, **options):
try:
cursor = connection.cursor()
self.stdout.write('Building SQL bins into database..\n')
for apps in get_apps():
sql_bin = os.path.join(os.path.dirname(apps.__file__), 'sql/bin')
if os.path.isdir(sql_bin):
self.stdout.write('SQL/BIN found in %s ...\n' % apps.__name__)
for file in glob.glob("%s/*.sql" % sql_bin):
self.stdout.write('Executing (%s) in %s\n' % (file, apps.__name__))
f = open(file, mode='r')
sql = f.read()
f.close()
cursor.execute(sql)
cursor.execute('commit')
self.stdout.write('\n')
self.stdout.write('Execution completed, exiting.\n')
except Exception as e:
raise CommandError('Exception: %s' % str(e))
def find_modules_for_script(script):
""" find script module which contains 'run' attribute """
modules = []
# first look in apps
for app in get_apps():
app_name = app.__name__.split(".")[:-1] # + ['fixtures']
for subdir in subdirs:
mod = my_import(".".join(app_name + [subdir, script]))
if mod:
modules.append(mod)
# try app.DIR.script import
sa = script.split(".")
for subdir in subdirs:
nn = ".".join(sa[:-1] + [subdir, sa[-1]])
mod = my_import(nn)
if mod:
modules.append(mod)
# try direct import
if script.find(".") != -1:
mod = my_import(script)
if mod:
modules.append(mod)
return modules
def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]):
"""Test runner to support many DocTests *.txt files and TestUnits *.py
using a setting TEST_FILES in app.tests module"""
for app in get_apps():
test_mod = get_tests(app)
if not test_mod or hasattr(test_mod, 'suite'):
continue
suites = []
# DocTest files
for filename in getattr(test_mod, 'DOCTEST_FILES', []):
try:
suites.append(doctest.DocFileSuite(
filename,
package=test_mod,
encoding='utf-8',
))
except TypeError:
suites.append(doctest.DocFileSuite(
filename,
package=test_mod,
))
# Unit Tests modules
for module in getattr(test_mod, 'UNITTEST_MODULES', []):
suites.append(unittest.TestLoader().loadTestsFromModule(module))
# Sets the 'suites' attribute to test module
if suites:
print suites
test_mod.suite = lambda: unittest.TestSuite(suites)
return old_run_tests(test_labels, verbosity, interactive, extra_tests)
def handle(self, *labels, **options):
if not labels or len(labels) > 1:
raise CommandError('Enter one directory name.')
label = labels[0]
final_dest = osp.join(os.getcwd(), label)
if osp.exists(final_dest):
raise CommandError('Directory already exists')
os.mkdir(final_dest)
apps = get_apps()
for x in apps:
app_dir = osp.dirname(x.__file__)
module = x.__name__
app = module.split('.')[-2]
if app == 'admin': continue
media_dir = osp.join(app_dir, "media", app)
if not osp.isdir(media_dir):
media_dir = osp.join(app_dir, "media")
if osp.exists(media_dir):
print "copy", media_dir, '->', osp.join(final_dest, app)
shutil.copytree(media_dir, osp.join(final_dest, app))
def _get_fixture_dirs(self):
"""Returns the list of fixture directories.
This is computed only once and cached.
"""
if not TestCase._fixture_dirs:
app_module_paths = []
for app in get_apps():
if hasattr(app, '__path__'):
# It's a 'models/' subpackage.
for path in app.__path__:
app_module_paths.append(path)
else:
# It's a models.py module
app_module_paths.append(app.__file__)
all_fixture_dirs = [
os.path.join(os.path.dirname(path), 'fixtures')
for path in app_module_paths
]
TestCase._fixture_dirs = [
fixture_dir
for fixture_dir in all_fixture_dirs
if os.path.exists(fixture_dir)
]
return TestCase._fixture_dirs
def handle(self, *args, **options):
fields_data = []
if len(args) != 0:
for arg in args:
field_spec = arg.split('.')
if len(field_spec) == 1:
app = get_app(field_spec[0])
models = get_models(app)
for model in models:
fields_data += all_fsm_fields_data(model)
elif len(field_spec) == 2:
model = get_model(field_spec[0], field_spec[1])
fields_data += all_fsm_fields_data(model)
elif len(field_spec) == 3:
model = get_model(field_spec[0], field_spec[1])
fields_data.append((model._meta.get_field_by_name(field_spec[2])[0], model))
else:
for app in get_apps():
for model in get_models(app):
fields_data += all_fsm_fields_data(model)
dotdata = generate_dot(fields_data)
if options['outputfile']:
self.render_output(dotdata, **options)
else:
print(dotdata)
def handle(self, *args, **options):
# this is overridden so we can also do the trick on couchapps
for app in get_apps():
couchdbkit_handler.copy_designs(app, temp='tmp', verbosity=2)
for plugin in get_preindex_plugins():
plugin.copy_designs(temp='tmp')
def run_tests(test_labels, verbosity=1, interactive=True,
extra_tests=[]):
"""
Test runner which displays a code coverage report at the end of the
run.
"""
django_test_runner = get_runner(settings)
coverage.use_cache(0)
coverage.start()
results = django_test_runner(test_labels, verbosity, interactive,
extra_tests)
coverage.stop()
coverage_modules = []
if test_labels:
for label in test_labels:
# Don't report coverage if you're only running a single
# test case.
if '.' not in label:
app = get_app(label)
coverage_modules.extend(get_all_coverage_modules(app))
else:
for app in get_apps():
coverage_modules.extend(get_all_coverage_modules(app))
if coverage_modules:
coverage.report(coverage_modules, show_missing=1)
return results
def runjobs_by_signals(self, when, options):
""" Run jobs from the signals """
# Thanks for Ian Holsman for the idea and code
from django_extensions.management import signals
from django.db import models
from django.conf import settings
verbosity = int(options.get('verbosity', 1))
for app_name in settings.INSTALLED_APPS:
try:
__import__(app_name + '.management', '', '', [''])
except ImportError:
pass
for app in models.get_apps():
if verbosity > 1:
app_name = '.'.join(app.__name__.rsplit('.')[:-1])
print("Sending %s job signal for: %s" % (when, app_name))
if when == 'minutely':
signals.run_minutely_jobs.send(sender=app, app=app)
elif when == 'quarter_hourly':
signals.run_quarter_hourly_jobs.send(sender=app, app=app)
elif when == 'hourly':
signals.run_hourly_jobs.send(sender=app, app=app)
elif when == 'daily':
signals.run_daily_jobs.send(sender=app, app=app)
elif when == 'weekly':
signals.run_weekly_jobs.send(sender=app, app=app)
elif when == 'monthly':
signals.run_monthly_jobs.send(sender=app, app=app)
elif when == 'yearly':
signals.run_yearly_jobs.send(sender=app, app=app)
def emit_post_migrate(verbosity, interactive, database):
# Emit the post migrate signal. This allows individual applications to
# respond as if the database had been migrated from scratch.
all_models = []
for app in models.get_apps():
all_models.extend(router.get_migratable_models(app, database, include_auto_created=True))
emit_post_migrate_signal(set(all_models), verbosity, interactive, database)
def handle_noargs(self, **options):
# retrieve sqlalchemy engine for selected database
db = options.get('database')
engine = get_engine(db)
# retrieve schema
schema = options.get('schema')
if schema is not None:
schema = Schema(schema) # account for setting overwrite if any
metadatas = set([])
for modelmodul in get_apps():
for n, m in inspect.getmembers(modelmodul):
if isinstance(m, MetaData):
# if o is bound to another db than engine, ignore it
if (m.bind is not None) and m.bind != engine:
continue
# if a schema was set
# ignore metadata that don't belong to it
if schema and m.schema != schema:
continue
metadatas.add(m)
for metadata in metadatas:
print("now synchronizing %s" % metadata)
metadata.create_all(bind=engine)
def dump_data(request,appname):
app_list = SortedDict()
try:
if request.POST:
for appname in request.POST.getlist('apps'):
app = get_app(appname)
app_list[app] = None
appname = 'choices'
else:
app = get_app(appname)
app_list[app] = None
except ImproperlyConfigured:
if appname == 'all':
for app in get_apps():
app_list[app] = None
if(len(app_list) > 0):
objects = []
for model in sort_dependencies(app_list.items()):
if not model._meta.proxy and router.allow_syncdb(DEFAULT_DB_ALIAS, model):
objects.extend(model._default_manager.using(DEFAULT_DB_ALIAS).all())
serializers.get_serializer('json')
json = serializers.serialize('json', objects, indent=2,use_natural_keys=True)
response = HttpResponse(json, mimetype='application/json');
response['Content-Disposition'] = 'attachment; filename=%s_%s_fixture.json' % (date.today().__str__(),appname)
return response
return render_to_response('diagnostic/dumpdata.html',context_instance=RequestContext(request))
def print_info(self):
from django.db.models import get_apps
app_list = get_apps()
# output.append("App list: %s" % app_list)
from django.core.management import sql
from django.core.management.color import no_style
def print_out(app, method_name):
method = getattr(sql, method_name)
try:
output = method(app, no_style())
except:
# sql_custom takes no stype... why?
output = method(app)
if output==[]:
return
print "--\n-- %s:\n--" % method_name
for line in output:
print line
for app in app_list:
print "--\n--",
print "_"*77
print "-- %s\n--" % app.__name__
print_out(app, "sql_create")
print_out(app, "sql_custom")
print_out(app, "sql_indexes")
class Command(NoArgsCommand):
option_list = syncdb.Command.option_list + (
make_option(
'--migrate',
action='store_true',
dest='migrate',
default=False,
help=
'Tells South to also perform migrations after the sync. Default for during testing, and other internal calls.'
),
make_option(
'--all',
action='store_true',
dest='migrate_all',
default=False,
help=
'Makes syncdb work on all apps, even migrated ones. Be careful!'),
)
if '--verbosity' not in [
opt.get_opt_string() for opt in syncdb.Command.option_list
]:
option_list += (make_option(
'--verbosity',
action='store',
dest='verbosity',
default='1',
type='choice',
choices=['0', '1', '2'],
help=
'Verbosity level; 0=minimal output, 1=normal output, 2=all output'
), )
help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created, except those which use migrations."
def handle_noargs(self, migrate_all=False, **options):
# Import the 'management' module within each installed app, to register
# dispatcher events.
# This is copied from Django, to fix bug #511.
try:
from django.utils.importlib import import_module
except ImportError:
pass # TODO: Remove, only for Django1.0
else:
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError, exc:
msg = exc.args[0]
if not msg.startswith(
'No module named') or 'management' not in msg:
raise
# Work out what uses migrations and so doesn't need syncing
apps_needing_sync = []
apps_migrated = []
for app in models.get_apps():
app_label = get_app_label(app)
if migrate_all:
apps_needing_sync.append(app_label)
else:
try:
migrations = migration.Migrations(app_label)
except NoMigrations:
# It needs syncing
apps_needing_sync.append(app_label)
else:
# This is a migrated app, leave it
apps_migrated.append(app_label)
verbosity = int(options.get('verbosity', 0))
# Run syncdb on only the ones needed
if verbosity:
print "Syncing..."
old_installed, settings.INSTALLED_APPS = settings.INSTALLED_APPS, apps_needing_sync
old_app_store, cache.app_store = cache.app_store, SortedDict([
(k, v) for (k, v) in cache.app_store.items()
if get_app_label(k) in apps_needing_sync
])
# This will allow the setting of the MySQL storage engine, for example.
for db in dbs.values():
db.connection_init()
# OK, run the actual syncdb
syncdb.Command().execute(**options)
settings.INSTALLED_APPS = old_installed
cache.app_store = old_app_store
# Migrate if needed
if options.get('migrate', True):
if verbosity:
print "Migrating..."
management.call_command('migrate', **options)
# Be obvious about what we did
if verbosity:
print "\nSynced:\n > %s" % "\n > ".join(apps_needing_sync)
if options.get('migrate', True):
if verbosity:
print "\nMigrated:\n - %s" % "\n - ".join(apps_migrated)
else:
if verbosity:
print "\nNot synced (use migrations):\n - %s" % "\n - ".join(
apps_migrated)
print "(use ./manage.py migrate to migrate these)"
def handle(self, *fixture_labels, **options):
from django.db.models import get_apps
from django.core import serializers
from django.db import connection, transaction
from django.conf import settings
self.style = no_style()
verbosity = int(options.get('verbosity', 1))
show_traceback = options.get('traceback', False)
# Keep a count of the installed objects and fixtures
fixture_count = 0
object_count = 0
models = set()
humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
app_fixtures = [
os.path.join(os.path.dirname(app.__file__), 'fixtures')
for app in get_apps()
]
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) == 1:
fixture_name = fixture_label
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats():
formats = [format]
else:
formats = []
if verbosity >= 2:
if formats:
print "Loading '%s' fixtures..." % fixture_name
else:
print "Skipping fixture '%s': %s is not a known serialization format" % (
fixture_name, format)
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(
settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity > 1:
print "Checking %s for fixtures..." % humanize(fixture_dir)
label_found = False
for format in formats:
serializer = serializers.get_serializer(format)
if verbosity > 1:
print "Trying %s for %s fixture '%s'..." % \
(humanize(fixture_dir), format, fixture_name)
try:
full_path = os.path.join(
fixture_dir, '.'.join([fixture_name, format]))
fixture = open(full_path, 'r')
if label_found:
fixture.close()
print self.style.ERROR(
"Multiple fixtures named '%s' in %s. Aborting."
% (fixture_name, humanize(fixture_dir)))
transaction.rollback()
transaction.leave_transaction_management()
return
else:
fixture_count += 1
if verbosity > 0:
print "Installing %s fixture '%s' from %s." % \
(format, fixture_name, humanize(fixture_dir))
try:
objects = serializers.deserialize(
format, fixture)
for obj in objects:
object_count += 1
models.add(obj.object.__class__)
obj.save()
label_found = True
except Exception, e:
fixture.close()
transaction.rollback()
transaction.leave_transaction_management()
if show_traceback:
import traceback
traceback.print_exc()
else:
sys.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s\n"
% (full_path, str(e))))
return
fixture.close()
except:
if verbosity >= 2:
print "No %s fixture '%s' in %s." % \
(format, fixture_name, humanize(fixture_dir))
if object_count > 0:
sequence_sql = connection.ops.sequence_reset_sql(
self.style, models)
if sequence_sql:
if verbosity > 1:
print "Resetting sequences"
for line in sequence_sql:
cursor.execute(line)
transaction.commit()
transaction.leave_transaction_management()
if object_count == 0:
if verbosity >= 2:
print "No fixtures found."
else:
if verbosity > 0:
print "Installed %d object(s) from %d fixture(s)" % (
object_count, fixture_count)
def handle(self, *app_labels, **options):
from django.db.models import get_app, get_apps, get_model
output_folder = options.get('output_folder')
print "Output folder:", output_folder
print "NOTE: See --output-folder option"
max_records_per_chunk = options.get('max_records_per_chunk')
format = options.get('format')
indent = options.get('indent')
using = options.get('database')
excludes = options.get('exclude')
show_traceback = options.get('traceback')
use_natural_keys = options.get('use_natural_keys')
use_base_manager = options.get('use_base_manager')
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
app_label, model_name = exclude.split('.', 1)
model_obj = get_model(app_label, model_name)
if not model_obj:
raise CommandError('Unknown model in excludes: %s' % exclude)
excluded_models.add(model_obj)
else:
try:
app_obj = get_app(exclude)
excluded_apps.add(app_obj)
except ImproperlyConfigured:
raise CommandError('Unknown app in excludes: %s' % exclude)
if len(app_labels) == 0:
app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps)
else:
app_list = SortedDict()
for label in app_labels:
try:
app_label, model_label = label.split('.')
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" % app_label)
if app in excluded_apps:
continue
model = get_model(app_label, model_label)
if model is None:
raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
if app in app_list.keys():
if app_list[app] and model not in app_list[app]:
app_list[app].append(model)
else:
app_list[app] = [model]
except ValueError:
# This is just an app - no model qualifier
app_label = label
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" % app_label)
if app in excluded_apps:
continue
app_list[app] = None
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
raise CommandError("Unknown serialization format: %s" % format)
try:
serializers.get_serializer(format)
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
# Now collate the objects to be serialized.
objects = []
model_count = 1000
chunk_count = 1000
for model in sort_dependencies(app_list.items()):
model_count += 1
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(using, model):
if use_base_manager:
objects.extend(model._base_manager.using(using).all())
else:
items_total = model._default_manager.using(using).count()
chunks_total = (items_total / max_records_per_chunk) +1
for chunk_num in range(0, chunks_total):
output_objects = model._default_manager.using(using).all().order_by('id')[chunk_num*max_records_per_chunk:(chunk_num+1)*max_records_per_chunk]
if output_objects:
chunk_count += 1
dump_file_name = output_folder + "/%d_%d.json" % (model_count, chunk_count)
print "Dumping file: %s [%d]" % (dump_file_name, chunks_total)
output = serializers.serialize(format, output_objects, indent=indent,
use_natural_keys=use_natural_keys)
with open(dump_file_name, "w") as dumpfile:
dumpfile.write(output)
return ''
def handle(self, *app_labels, **options):
from django.db.models import get_app, get_apps, get_model
format = options.get('format')
indent = options.get('indent')
using = options.get('database')
excludes = options.get('exclude')
show_traceback = options.get('traceback')
use_natural_keys = options.get('use_natural_keys')
use_base_manager = options.get('use_base_manager')
pks = options.get('primary_keys')
if pks:
primary_keys = pks.split(',')
else:
primary_keys = []
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
app_label, model_name = exclude.split('.', 1)
model_obj = get_model(app_label, model_name)
if not model_obj:
raise CommandError('Unknown model in excludes: %s' %
exclude)
excluded_models.add(model_obj)
else:
try:
app_obj = get_app(exclude)
excluded_apps.add(app_obj)
except ImproperlyConfigured:
raise CommandError('Unknown app in excludes: %s' % exclude)
if len(app_labels) == 0:
if primary_keys:
raise CommandError(
"You can only use --pks option with one model")
app_list = OrderedDict(
(app, None) for app in get_apps() if app not in excluded_apps)
else:
if len(app_labels) > 1 and primary_keys:
raise CommandError(
"You can only use --pks option with one model")
app_list = OrderedDict()
for label in app_labels:
try:
app_label, model_label = label.split('.')
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" %
app_label)
if app in excluded_apps:
continue
model = get_model(app_label, model_label)
if model is None:
raise CommandError("Unknown model: %s.%s" %
(app_label, model_label))
if app in app_list.keys():
if app_list[app] and model not in app_list[app]:
app_list[app].append(model)
else:
app_list[app] = [model]
except ValueError:
if primary_keys:
raise CommandError(
"You can only use --pks option with one model")
# This is just an app - no model qualifier
app_label = label
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" %
app_label)
if app in excluded_apps:
continue
app_list[app] = None
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
raise CommandError("Unknown serialization format: %s" % format)
try:
serializers.get_serializer(format)
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
def get_objects():
# Collate the objects to be serialized.
for model in sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(
using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(
model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
for obj in queryset.iterator():
yield obj
try:
self.stdout.ending = None
serializers.serialize(format,
get_objects(),
indent=indent,
use_natural_keys=use_natural_keys,
stream=self.stdout)
except Exception as e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
def handle_noargs(self, migrate_all=False, **options):
# Import the 'management' module within each installed app, to register
# dispatcher events.
# This is copied from Django, to fix bug #511.
try:
from django.utils.importlib import import_module
except ImportError:
pass # TODO: Remove, only for Django1.0
else:
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError as exc:
msg = exc.args[0]
if not msg.startswith('No module named') or 'management' not in msg:
raise
# Work out what uses migrations and so doesn't need syncing
apps_needing_sync = []
apps_migrated = []
for app in models.get_apps():
app_label = get_app_label(app)
if migrate_all:
apps_needing_sync.append(app_label)
else:
try:
migrations = migration.Migrations(app_label)
except NoMigrations:
# It needs syncing
apps_needing_sync.append(app_label)
else:
# This is a migrated app, leave it
apps_migrated.append(app_label)
verbosity = int(options.get('verbosity', 0))
# Run syncdb on only the ones needed
if verbosity:
print("Syncing...")
old_installed, settings.INSTALLED_APPS = settings.INSTALLED_APPS, apps_needing_sync
old_app_store, cache.app_store = cache.app_store, SortedDict([
(k, v) for (k, v) in cache.app_store.items()
if get_app_label(k) in apps_needing_sync
])
# This will allow the setting of the MySQL storage engine, for example.
for db in dbs.values():
db.connection_init()
# OK, run the actual syncdb
syncdb.Command().execute(**options)
settings.INSTALLED_APPS = old_installed
cache.app_store = old_app_store
# Migrate if needed
if options.get('migrate', True):
if verbosity:
print("Migrating...")
# convert from store_true to store_false
options['no_initial_data'] = not options.get('load_initial_data', True)
management.call_command('migrate', **options)
# Be obvious about what we did
if verbosity:
print("\nSynced:\n > %s" % "\n > ".join(apps_needing_sync))
if options.get('migrate', True):
if verbosity:
print("\nMigrated:\n - %s" % "\n - ".join(apps_migrated))
else:
if verbosity:
print("\nNot synced (use migrations):\n - %s" % "\n - ".join(apps_migrated))
print("(use ./manage.py migrate to migrate these)")
class Command(NoArgsCommand):
"""
It validates all models that are G11nBase subclass.
"""
help = "Validates all models that are G11nBase subclass."
def handle(self, *args, **options):
for app_name in settings.INSTALLED_APPS: #controlla tutte le app in INSTALLED_APP
try:
import_module('.management',
app_name) # Prova ad importare l'applicazione
except ImportError, exc:
msg = exc.args[0]
if not msg.startswith(
'No module named'
) or 'management' not in msg: #se non ci riesce lancia eccezione
raise
for app in models.get_apps(): # cicla tutte le applicazioni
for m in models.get_models(
app): # per ogni modello nell'applicazione corrente
if issubclass(
m, G11nModel
): # se e' subclass di G11nModel deve controllare che estende una classe astratta e non conreta
for base in m.__bases__:
if issubclass(base,
G11nModel) and not base._meta.abstract:
raise CommandError(
"Error in %s.%s . G11nModel can extend only abstract Model"
% (m.__module__, m.__name__))
elif issubclass(
m,
G11nBase): # se il modello e' sottoclasse di G11nBase
if not hasattr(
m.G11nMeta, "g11n"
): #controlla che abbia il parametro g11n nella meta class G11nMeta
raise CommandError(
"Error in %s.%s . G11nMeta must have field with the name g11n"
% (m.__module__, m.__name__))
elif not hasattr(
m.G11nMeta, "fieldname"
): #controlla che abbia il parametro fieldname nella meta class G11nMeta
raise CommandError(
"Error in %s.%s . G11nMeta must have field with the name fieldname"
% (m.__module__, m.__name__))
elif not m.G11nMeta.g11n: #controlla che il parametro g11n non sia nullo nella meta class G11nMeta
raise CommandError(
"Error in %s. %s is a G11nBase model and it needs a meta class G11nMeta."
% (m.__module__, m.__name__))
elif not hasattr(
app, m.G11nMeta.g11n
): #controlla che esista il modello definito nel parametro g11n della meta class G11nMeta
raise CommandError(
"Error in %s Model with the name %s doesn't exists."
% (m.__module__, m.G11nMeta.g11n))
else:
g11nmodel = getattr(
app, m.G11nMeta.g11n
) #importa il modello ereditato da G11nModel
if not issubclass(
g11nmodel, G11nModel
): #controlla che effettivamente sia un G11nModel
raise CommandError(
"Error in %s . %s is not a G11nModel subclass."
% (m.__module__, g11nmodel.__name__))
else: #se G11nBase passa la validazione e siamo in un G11nModel allora controlla le sue configurazioni
foreign_keys = []
properties = []
for k, v in m.__dict__.items():
if type(v) == property:
properties.append(k)
for field in g11nmodel._meta.fields: #per ogni field del g11nmodel
if field.get_internal_type(
) == "ForeignKey" and m.__name__ == field.rel.to.__name__:
# se il field e' una FK il nome del modello G11nBase == al nome del modello al quale g11nmodel si riferisce per questo field
foreign_keys.append(
field
) # aggiunge alla lista questo campo
if m.G11nMeta.fieldname != field.name:
#se il nome del field definito nella meta class G11nMeta != dal nome di questo campo allora lancia eccezione
raise CommandError(
"Error in %s.%s . G11nMeta.fieldname must be like a field's name in %s."
% (m.__module__, m.__name__,
g11nmodel.__name__))
if field.name in properties:
raise CommandError(
"Error in %s . %s has the property '%s' with the same name of a %s's attribute."
% (m.__module__, m.__name__,
field.name, g11nmodel.__name__))
if not foreign_keys: # se la lista e' vuota, significa che il g11nmodel non ha una FK verso il G11nBase
raise CommandError(
"Error in %s . %s has not a field related to %s."
% (m.__module__, g11nmodel.__name__,
m.__name__))
self.stdout.write("0 errors found in all G11nBase models\n")
def handle(self, *args, **options):
"""Handle the command."""
if len(args) != 1:
raise CommandError("You must specify a filename on the command "
"line.")
filename = args[0]
if not os.path.exists(filename):
raise CommandError("%s does not exist." % filename)
try:
import django_reset
except ImportError:
raise CommandError("Before using this command, you need to "
"install the 'django-reset' package")
confirm = input("""
This will wipe out your existing database prior to loading. It is highly
recommended that you have a full SQL database dump in case things go wrong.
You should only use this if you're migrating from one type of database to
another, with the same version of Review Board on each.
Are you sure you want to continue?"
Type 'yes' to continue, or 'no' to cancel: """)
if confirm != 'yes':
return
apps = [app.__name__.split('.')[-2] for app in get_apps()]
os.system('./reviewboard/manage.py reset --noinput %s' %
' '.join(apps))
transaction_setup = False
try:
with open(filename, 'r') as f:
line = f.readline()
m = re.match("^# dbdump v(\d+) - (\d+) objects$", line)
if not m:
raise CommandError("Unknown dump format\n")
version = int(m.group(1))
totalobjs = int(m.group(2))
i = 0
prev_pct = -1
if version != 1:
raise CommandError("Unknown dump version\n")
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
transaction_setup = True
self.stdout.write("Importing new style dump format (v%s)" %
version)
for line in f:
if line[0] == "{":
for obj in serializers.deserialize(
"json", "[%s]" % line):
try:
obj.save()
except Exception as e:
self.stderr.write("Error: %s\n" % e)
self.stderr.write("Line %s: '%s'" % (i, line))
elif line[0] != "#":
self.stderr.write("Junk data on line %s" % i)
db.reset_queries()
i += 1
pct = (i * 100 / totalobjs)
if pct != prev_pct:
self.stdout.write(" [%s%%]\r" % pct)
self.stdout.flush()
prev_pct = pct
transaction.commit()
transaction.leave_transaction_management()
except Exception as e:
raise CommandError("Problem installing '%s': %s\n" % (filename, e))
if transaction_setup:
transaction.rollback()
transaction.leave_transaction_management()
self.stdout.write('\nDone.')
def validate(cls, model):
"""
Does basic ModelAdmin option validation. Calls custom validation
classmethod in the end if it is provided in cls. The signature of the
custom validation classmethod should be: def validate(cls, model).
"""
# Before we can introspect models, they need to be fully loaded so that
# inter-relations are set up correctly. We force that here.
models.get_apps()
opts = model._meta
validate_base(cls, model)
# list_display
if hasattr(cls, 'list_display'):
check_isseq(cls, 'list_display', cls.list_display)
for idx, field in enumerate(cls.list_display):
if not callable(field):
if not hasattr(cls, field):
if not hasattr(model, field):
try:
opts.get_field(field)
except models.FieldDoesNotExist:
raise ImproperlyConfigured(
"%s.list_display[%d], %r is not a callable or an attribute of %r or found in the model %r."
% (cls.__name__, idx, field, cls.__name__,
model._meta.object_name))
else:
# getattr(model, field) could be an X_RelatedObjectsDescriptor
f = fetch_attr(cls, model, opts,
"list_display[%d]" % idx, field)
if isinstance(f, models.ManyToManyField):
raise ImproperlyConfigured(
"'%s.list_display[%d]', '%s' is a ManyToManyField which is not supported."
% (cls.__name__, idx, field))
# list_display_links
if hasattr(cls, 'list_display_links'):
check_isseq(cls, 'list_display_links', cls.list_display_links)
for idx, field in enumerate(cls.list_display_links):
fetch_attr(cls, model, opts, 'list_display_links[%d]' % idx, field)
if field not in cls.list_display:
raise ImproperlyConfigured(
"'%s.list_display_links[%d]'"
"refers to '%s' which is not defined in 'list_display'." %
(cls.__name__, idx, field))
# list_filter
if hasattr(cls, 'list_filter'):
check_isseq(cls, 'list_filter', cls.list_filter)
for idx, field in enumerate(cls.list_filter):
get_field(cls, model, opts, 'list_filter[%d]' % idx, field)
# list_per_page = 100
if hasattr(cls,
'list_per_page') and not isinstance(cls.list_per_page, int):
raise ImproperlyConfigured("'%s.list_per_page' should be a integer." %
cls.__name__)
# list_editable
if hasattr(cls, 'list_editable') and cls.list_editable:
check_isseq(cls, 'list_editable', cls.list_editable)
for idx, field_name in enumerate(cls.list_editable):
try:
field = opts.get_field_by_name(field_name)[0]
except models.FieldDoesNotExist:
raise ImproperlyConfigured(
"'%s.list_editable[%d]' refers to a "
"field, '%s', not defined on %s." %
(cls.__name__, idx, field_name, model.__name__))
if field_name not in cls.list_display:
raise ImproperlyConfigured(
"'%s.list_editable[%d]' refers to "
"'%s' which is not defined in 'list_display'." %
(cls.__name__, idx, field_name))
if field_name in cls.list_display_links:
raise ImproperlyConfigured(
"'%s' cannot be in both '%s.list_editable'"
" and '%s.list_display_links'" %
(field_name, cls.__name__, cls.__name__))
if not cls.list_display_links and cls.list_display[
0] in cls.list_editable:
raise ImproperlyConfigured(
"'%s.list_editable[%d]' refers to"
" the first field in list_display, '%s', which can't be"
" used unless list_display_links is set." %
(cls.__name__, idx, cls.list_display[0]))
if not field.editable:
raise ImproperlyConfigured(
"'%s.list_editable[%d]' refers to a "
"field, '%s', which isn't editable through the admin." %
(cls.__name__, idx, field_name))
# search_fields = ()
if hasattr(cls, 'search_fields'):
check_isseq(cls, 'search_fields', cls.search_fields)
# date_hierarchy = None
if cls.date_hierarchy:
f = get_field(cls, model, opts, 'date_hierarchy', cls.date_hierarchy)
if not isinstance(f, (models.DateField, models.DateTimeField)):
raise ImproperlyConfigured(
"'%s.date_hierarchy is "
"neither an instance of DateField nor DateTimeField." %
cls.__name__)
# ordering = None
if cls.ordering:
check_isseq(cls, 'ordering', cls.ordering)
for idx, field in enumerate(cls.ordering):
if field == '?' and len(cls.ordering) != 1:
raise ImproperlyConfigured(
"'%s.ordering' has the random "
"ordering marker '?', but contains other fields as "
"well. Please either remove '?' or the other fields." %
cls.__name__)
if field == '?':
continue
if field.startswith('-'):
field = field[1:]
# Skip ordering in the format field1__field2 (FIXME: checking
# this format would be nice, but it's a little fiddly).
if '__' in field:
continue
get_field(cls, model, opts, 'ordering[%d]' % idx, field)
# list_select_related = False
# save_as = False
# save_on_top = False
for attr in ('list_select_related', 'save_as', 'save_on_top'):
if not isinstance(getattr(cls, attr), bool):
raise ImproperlyConfigured("'%s.%s' should be a boolean." %
(cls.__name__, attr))
# inlines = []
if hasattr(cls, 'inlines'):
check_isseq(cls, 'inlines', cls.inlines)
for idx, inline in enumerate(cls.inlines):
if not issubclass(inline, BaseModelAdmin):
raise ImproperlyConfigured("'%s.inlines[%d]' does not inherit "
"from BaseModelAdmin." %
(cls.__name__, idx))
if not inline.model:
raise ImproperlyConfigured("'model' is a required attribute "
"of '%s.inlines[%d]'." %
(cls.__name__, idx))
if not issubclass(inline.model, models.Model):
raise ImproperlyConfigured("'%s.inlines[%d].model' does not "
"inherit from models.Model." %
(cls.__name__, idx))
validate_base(inline, inline.model)
validate_inline(inline, cls, model)
def handle(self, *fixture_labels, **options):
from django.db.models import get_apps
from django.core import serializers
from django.db import connection, transaction
from django.conf import settings
self.style = no_style()
verbosity = int(options.get('verbosity', 1))
show_traceback = options.get('traceback', False)
# commit is a stealth option - it isn't really useful as
# a command line option, but it can be useful when invoking
# loaddata from within another script.
# If commit=True, loaddata will use its own transaction;
# if commit=False, the data load SQL will become part of
# the transaction in place when loaddata was invoked.
commit = options.get('commit', True)
# Keep a count of the installed objects and fixtures
fixture_count = 0
object_count = 0
models = set()
humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
if commit:
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
self.disable_forward_ref_checks()
class SingleZipReader(zipfile.ZipFile):
def __init__(self, *args, **kwargs):
zipfile.ZipFile.__init__(self, *args, **kwargs)
if settings.DEBUG:
assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file."
def read(self):
return zipfile.ZipFile.read(self, self.namelist()[0])
compression_types = {
None: file,
'gz': gzip.GzipFile,
'zip': SingleZipReader
}
if has_bz2:
compression_types['bz2'] = bz2.BZ2File
app_fixtures = [os.path.join(os.path.dirname(app.__file__), 'fixtures') for app in get_apps()]
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) > 1 and parts[-1] in compression_types:
compression_formats = [parts[-1]]
parts = parts[:-1]
else:
compression_formats = compression_types.keys()
if len(parts) == 1:
fixture_name = parts[0]
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats():
formats = [format]
else:
formats = []
if formats:
if verbosity > 1:
print("Loading '%s' fixtures..." % fixture_name)
else:
self.enable_forward_ref_checks(cursor)
sys.stderr.write(
self.style.ERROR("Problem installing fixture '%s': %s is not a known serialization format." %
(fixture_name, format)))
transaction.rollback()
transaction.leave_transaction_management()
return
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity > 1:
print("Checking %s for fixtures..." % humanize(fixture_dir))
label_found = False
for format in formats:
for compression_format in compression_formats:
if compression_format:
file_name = '.'.join([fixture_name, format,
compression_format])
else:
file_name = '.'.join([fixture_name, format])
if verbosity > 1:
print("Trying %s for %s fixture '%s'..." % \
(humanize(fixture_dir), file_name, fixture_name))
full_path = os.path.join(fixture_dir, file_name)
open_method = compression_types[compression_format]
try:
fixture = open_method(full_path, 'r')
if label_found:
fixture.close()
self.enable_forward_ref_checks(cursor)
print(self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting." %
(fixture_name, humanize(fixture_dir))))
transaction.rollback()
transaction.leave_transaction_management()
return
else:
fixture_count += 1
objects_in_fixture = 0
if verbosity > 0:
print("Installing %s fixture '%s' from %s." % \
(format, fixture_name, humanize(fixture_dir)))
try:
objects = serializers.deserialize(format, fixture)
for obj in objects:
objects_in_fixture += 1
self.handle_ref_checks(cursor, obj)
models.add(obj.object.__class__)
obj.save()
object_count += objects_in_fixture
label_found = True
except (SystemExit, KeyboardInterrupt):
self.enable_forward_ref_checks(cursor)
raise
except Exception:
import traceback
fixture.close()
self.enable_forward_ref_checks(cursor)
transaction.rollback()
transaction.leave_transaction_management()
if show_traceback:
traceback.print_exc()
else:
sys.stderr.write(
self.style.ERROR("Problem installing fixture '%s': %s\n" %
(full_path, ''.join(traceback.format_exception(sys.exc_type,
sys.exc_value, sys.exc_traceback)))))
return
fixture.close()
# If the fixture we loaded contains 0 objects, assume that an
# error was encountered during fixture loading.
if objects_in_fixture == 0:
self.enable_forward_ref_checks(cursor)
sys.stderr.write(
self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)" %
(fixture_name)))
transaction.rollback()
transaction.leave_transaction_management()
return
except Exception as e:
if verbosity > 1:
print("No %s fixture '%s' in %s." % \
(format, fixture_name, humanize(fixture_dir)))
self.enable_forward_ref_checks(cursor)
# If we found even one object in a fixture, we need to reset the
# database sequences.
if object_count > 0:
sequence_sql = connection.ops.sequence_reset_sql(self.style, models)
if sequence_sql:
if verbosity > 1:
print("Resetting sequences")
for line in sequence_sql:
cursor.execute(line)
if commit:
transaction.commit()
transaction.leave_transaction_management()
if object_count == 0:
if verbosity > 1:
print("No fixtures found.")
else:
if verbosity > 0:
print("Installed %d object(s) from %d fixture(s)" % (object_count, fixture_count))
# Close the DB connection. This is required as a workaround for an
# edge case in MySQL: if the same connection is used to
# create tables, load data, and query, the query can return
# incorrect results. See Django #7572, MySQL #37735.
if commit:
connection.close()
def handle(self, *app_labels, **options):
from django.db.models import get_app, get_apps, get_model
format = options.get('format')
indent = options.get('indent')
using = options.get('database')
excludes = options.get('exclude')
show_traceback = options.get('traceback')
use_natural_keys = options.get('use_natural_keys')
use_base_manager = options.get('use_base_manager')
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
app_label, model_name = exclude.split('.', 1)
model_obj = get_model(app_label, model_name)
if not model_obj:
raise CommandError('Unknown model in excludes: %s' % exclude)
excluded_models.add(model_obj)
else:
try:
app_obj = get_app(exclude)
excluded_apps.add(app_obj)
except ImproperlyConfigured:
raise CommandError('Unknown app in excludes: %s' % exclude)
if len(app_labels) == 0:
app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps)
else:
app_list = SortedDict()
for label in app_labels:
try:
app_label, model_label = label.split('.')
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" % app_label)
if app in excluded_apps:
continue
model = get_model(app_label, model_label)
if model is None:
raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
if app in app_list.keys():
if app_list[app] and model not in app_list[app]:
app_list[app].append(model)
else:
app_list[app] = [model]
except ValueError:
# This is just an app - no model qualifier
app_label = label
try:
app = get_app(app_label)
except ImproperlyConfigured:
raise CommandError("Unknown application: %s" % app_label)
if app in excluded_apps:
continue
app_list[app] = None
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
raise CommandError("Unknown serialization format: %s" % format)
try:
serializers.get_serializer(format)
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
# Now collate the objects to be serialized.
objects = []
for model in sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_syncdb(using, model):
if use_base_manager:
objects.extend(model._base_manager.using(using).all())
else:
objects.extend(model._default_manager.using(using).all())
try:
return serializers.serialize(format, objects, indent=indent,
use_natural_keys=use_natural_keys)
except Exception, e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
class Command(NoArgsCommand):
option_list = NoArgsCommand.option_list + (
make_option(
'--noinput',
action='store_false',
dest='interactive',
default=True,
help='Tells Django to NOT prompt the user for input of any kind.'),
make_option('--database',
action='store',
dest='database',
default=DEFAULT_DB_ALIAS,
help='Nominates a database to synchronize. '
'Defaults to the "default" database.'),
)
help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created."
def handle_noargs(self, **options):
verbosity = int(options.get('verbosity', 1))
interactive = options.get('interactive')
show_traceback = options.get('traceback', False)
# Stealth option -- 'load_initial_data' is used by the testing setup
# process to disable initial fixture loading.
load_initial_data = options.get('load_initial_data', True)
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError, exc:
# This is slightly hackish. We want to ignore ImportErrors
# if the "management" module itself is missing -- but we don't
# want to ignore the exception if the management module exists
# but raises an ImportError for some reason. The only way we
# can do this is to check the text of the exception. Note that
# we're a bit broad in how we check the text, because different
# Python implementations may not use the same text.
# CPython uses the text "No module named management"
# PyPy uses "No module named myproject.myapp.management"
msg = exc.args[0]
if not msg.startswith(
'No module named') or 'management' not in msg:
raise
db = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[db]
cursor = connection.cursor()
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Build the manifest of apps and models that are to be synchronized
all_models = [(app.__name__.split('.')[-2], [
m for m in models.get_models(app, include_auto_created=True)
if router.allow_syncdb(db, m)
]) for app in models.get_apps()]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = SortedDict((app_name, filter(model_installed, model_list))
for app_name, model_list in all_models)
# Create the tables for each model
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if verbosity >= 2:
print "Processing %s.%s model" % (app_name,
model._meta.object_name)
sql, references = connection.creation.sql_create_model(
model, self.style, seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, self.style, pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, self.style, pending_references))
if verbosity >= 1 and sql:
print "Creating table %s" % model._meta.db_table
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
transaction.commit_unless_managed(using=db)
# Send the post_syncdb signal, so individual apps can do whatever they need
# to do at this point.
emit_post_sync_signal(created_models, verbosity, interactive, db)
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
# Install custom SQL for the app (but only if this
# is a model we've just created)
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(model, self.style,
connection)
if custom_sql:
if verbosity >= 1:
print "Installing custom SQL for %s.%s model" % (
app_name, model._meta.object_name)
try:
for sql in custom_sql:
cursor.execute(sql)
except Exception, e:
sys.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
(app_name, model._meta.object_name, e))
if show_traceback:
import traceback
traceback.print_exc()
transaction.rollback_unless_managed(using=db)
else:
transaction.commit_unless_managed(using=db)
else:
if verbosity >= 2:
print "No custom SQL for %s.%s model" % (
app_name, model._meta.object_name)
def generate_dot(app_labels, **kwargs):
disable_fields = kwargs.get('disable_fields', False)
include_models = parse_file_or_list(kwargs.get('include_models', ""))
all_applications = kwargs.get('all_applications', False)
use_subgraph = kwargs.get('group_models', False)
verbose_names = kwargs.get('verbose_names', False)
inheritance = kwargs.get('inheritance', False)
language = kwargs.get('language', None)
if language is not None:
activate_language(language)
exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', ""))
exclude_models = parse_file_or_list(kwargs.get('exclude_models', ""))
def skip_field(field):
if exclude_columns:
if verbose_names and field.verbose_name:
if field.verbose_name in exclude_columns:
return True
if field.name in exclude_columns:
return True
return False
t = loader.get_template('django_extensions/graph_models/head.html')
c = Context({})
dot = t.render(c)
apps = []
if all_applications:
apps = models.get_apps()
for app_label in app_labels:
app = models.get_app(app_label)
if not app in apps:
apps.append(app)
graphs = []
for app in apps:
graph = Context({
'name':
'"%s"' % app.__name__,
'app_name':
"%s" % '.'.join(app.__name__.split('.')[:-1]),
'cluster_app_name':
"cluster_%s" % app.__name__.replace(".", "_"),
'disable_fields':
disable_fields,
'use_subgraph':
use_subgraph,
'models': []
})
appmodels = get_models(app)
abstract_models = []
for appmodel in appmodels:
abstract_models = abstract_models + [
abstract_model
for abstract_model in appmodel.__bases__ if hasattr(
abstract_model, '_meta') and abstract_model._meta.abstract
]
abstract_models = list(set(abstract_models)) # remove duplicates
appmodels = abstract_models + appmodels
for appmodel in appmodels:
appmodel_abstracts = [
abstract_model.__name__
for abstract_model in appmodel.__bases__
if hasattr(abstract_model, '_meta')
and abstract_model._meta.abstract
]
# collect all attribs of abstract superclasses
def getBasesAbstractFields(c):
_abstract_fields = []
for e in c.__bases__:
if hasattr(e, '_meta') and e._meta.abstract:
_abstract_fields.extend(e._meta.fields)
_abstract_fields.extend(getBasesAbstractFields(e))
return _abstract_fields
abstract_fields = getBasesAbstractFields(appmodel)
model = {
'app_name': appmodel.__module__.replace(".", "_"),
'name': appmodel.__name__,
'abstracts': appmodel_abstracts,
'fields': [],
'relations': []
}
# consider given model name ?
def consider(model_name):
if exclude_models and model_name in exclude_models:
return False
return not include_models or model_name in include_models
if not consider(appmodel._meta.object_name):
continue
if verbose_names and appmodel._meta.verbose_name:
model['label'] = appmodel._meta.verbose_name
else:
model['label'] = model['name']
# model attributes
def add_attributes(field):
if verbose_names and field.verbose_name:
label = field.verbose_name
else:
label = field.name
t = type(field).__name__
if isinstance(field, (OneToOneField, ForeignKey)):
t += " ({0})".format(field.rel.field_name)
# TODO: ManyToManyField, GenericRelation
model['fields'].append({
'name': field.name,
'label': label,
'type': t,
'blank': field.blank,
'abstract': field in abstract_fields,
})
# Find all the real attributes. Relations are depicted as graph edges instead of attributes
attributes = [
field for field in appmodel._meta.local_fields
if not isinstance(field, RelatedField)
]
# find primary key and print it first, ignoring implicit id if other pk exists
pk = appmodel._meta.pk
if not appmodel._meta.abstract and pk in attributes:
add_attributes(pk)
for field in attributes:
if skip_field(field):
continue
if not field.primary_key:
add_attributes(field)
# FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph?
#if appmodel._meta.many_to_many:
# for field in appmodel._meta.many_to_many:
# if skip_field(field):
# continue
# add_attributes(field)
# relations
def add_relation(field, extras=""):
if verbose_names and field.verbose_name:
label = field.verbose_name
else:
label = field.name
# show related field name
if hasattr(field, 'related_query_name'):
label += ' (%s)' % field.related_query_name()
# handle self-relationships
if field.rel.to == 'self':
target_model = field.model
else:
target_model = field.rel.to
_rel = {
'target_app': target_model.__module__.replace('.', '_'),
'target': target_model.__name__,
'type': type(field).__name__,
'name': field.name,
'label': label,
'arrows': extras,
'needs_node': True
}
if _rel not in model['relations'] and consider(_rel['target']):
model['relations'].append(_rel)
for field in appmodel._meta.local_fields:
if field.attname.endswith(
'_ptr_id'
): # excluding field redundant with inheritance relation
continue
if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields
continue
if skip_field(field):
continue
if isinstance(field, OneToOneField):
add_relation(field, '[arrowhead=none, arrowtail=none]')
elif isinstance(field, ForeignKey):
add_relation(field, '[arrowhead=none, arrowtail=dot]')
for field in appmodel._meta.local_many_to_many:
if skip_field(field):
continue
if isinstance(field, ManyToManyField):
if (getattr(field, 'creates_table', False)
or # django 1.1.
(hasattr(field.rel.through, '_meta') and
field.rel.through._meta.auto_created)): # django 1.2
add_relation(
field, '[arrowhead=dot arrowtail=dot, dir=both]')
elif isinstance(field, GenericRelation):
add_relation(
field,
mark_safe(
'[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]'
))
if inheritance:
# add inheritance arrows
for parent in appmodel.__bases__:
if hasattr(parent, "_meta"): # parent is a model
l = "multi-table"
if parent._meta.abstract:
l = "abstract"
if appmodel._meta.proxy:
l = "proxy"
l += r"\ninheritance"
_rel = {
'target_app': parent.__module__.replace(".", "_"),
'target': parent.__name__,
'type': "inheritance",
'name': "inheritance",
'label': l,
'arrows': '[arrowhead=empty, arrowtail=none]',
'needs_node': True
}
# TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes.
if _rel not in model['relations'] and consider(
_rel['target']):
model['relations'].append(_rel)
graph['models'].append(model)
graphs.append(graph)
nodes = []
for graph in graphs:
nodes.extend([e['name'] for e in graph['models']])
for graph in graphs:
# don't draw duplication nodes because of relations
for model in graph['models']:
for relation in model['relations']:
if relation['target'] in nodes:
relation['needs_node'] = False
# render templates
t = loader.get_template('django_extensions/graph_models/body.html')
dot += '\n' + t.render(graph)
for graph in graphs:
t = loader.get_template('django_extensions/graph_models/rel.html')
dot += '\n' + t.render(graph)
t = loader.get_template('django_extensions/graph_models/tail.html')
c = Context({})
dot += '\n' + t.render(c)
return dot
def handle(self, *fixture_labels, **options):
'''
All database code is unnecessary but remove it will broke.
'''
#TODO remove database, connection code and stuff
using = options.get('database')
connection = connections[using]
self.style = no_style()
if not len(fixture_labels):
print("No database fixture specified. Please provide the path of at least one fixture in the command line.\n")
return
verbosity = int(options.get('verbosity'))
show_traceback = options.get('traceback')
# commit is a stealth option - it isn't really useful as
# a command line option, but it can be useful when invoking
# loaddata from within another script.
# If commit=True, loaddata will use its own transaction;
# if commit=False, the data load SQL will become part of
# the transaction in place when loaddata was invoked.
commit = options.get('commit', True)
# Keep a count of the installed objects and fixtures
fixture_count = 0
loaded_object_count = 0
fixture_object_count = 0
models = set()
humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
if commit:
transaction.commit_unless_managed(using=using)
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
class SingleZipReader(zipfile.ZipFile):
def __init__(self, *args, **kwargs):
zipfile.ZipFile.__init__(self, *args, **kwargs)
if settings.DEBUG:
assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file."
def read(self):
return zipfile.ZipFile.read(self, self.namelist()[0])
compression_types = {
None: open,
'gz': gzip.GzipFile,
'zip': SingleZipReader
}
if has_bz2:
compression_types['bz2'] = bz2.BZ2File
app_module_paths = []
for app in get_apps():
if hasattr(app, '__path__'):
# It's a 'models/' subpackage
for path in app.__path__:
app_module_paths.append(path)
else:
# It's a models.py module
app_module_paths.append(app.__file__)
app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths]
instances = []
apps = set()
try:
#with connection.constraint_checks_disabled():
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) > 1 and parts[-1] in compression_types:
compression_formats = [parts[-1]]
parts = parts[:-1]
else:
compression_formats = compression_types.keys()
if len(parts) == 1:
fixture_name = parts[0]
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats():
formats = [format]
else:
formats = []
if formats:
if verbosity >= 2:
print("Loading '%s' fixtures...\n" % fixture_name)
else:
print("Problem installing fixture '%s': %s is not a known serialization format.\n" %
(fixture_name, format))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(using=using)
return
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity >= 2:
self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir))
label_found = False
for combo in product([using, None], formats, compression_formats):
database, format, compression_format = combo
file_name = '.'.join(
p for p in [
fixture_name, database, format, compression_format
]
if p
)
if verbosity >= 3:
self.stdout.write("Trying %s for %s fixture '%s'...\n" % \
(humanize(fixture_dir), file_name, fixture_name))
full_path = os.path.join(fixture_dir, file_name)
open_method = compression_types[compression_format]
try:
fixture = open_method(full_path, 'r')
except IOError:
if verbosity >= 2:
self.stdout.write("No %s fixture '%s' in %s.\n" % \
(format, fixture_name, humanize(fixture_dir)))
else:
try:
if label_found:
print("Multiple fixtures named '%s' in %s. Aborting.\n" %
(fixture_name, humanize(fixture_dir)))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(using=using)
return
fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
if verbosity >= 2:
self.stdout.write("Installing %s fixture '%s' from %s.\n" % \
(format, fixture_name, humanize(fixture_dir)))
objects = serializers.deserialize(format, fixture, using=using)
for obj in objects:
objects_in_fixture += 1
if router.allow_syncdb(using, obj.object.__class__):
loaded_objects_in_fixture += 1
models.add(obj.object.__class__)
try:
obj.object.save = partial(memory_save, obj.object)
#if obj.object.__class__.__name__ == "PedidoExame":
# import ipdb; ipdb.set_trace() ### XXX BREAKPOINT
obj.object.save()
instances.append(obj)
except (DatabaseError, IntegrityError), e:
msg = "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
'app_label': obj.object._meta.app_label,
'object_name': obj.object._meta.object_name,
'pk': obj.object.pk,
'error_msg': e
}
raise e.__class__, e.__class__(msg), sys.exc_info()[2]
loaded_object_count += loaded_objects_in_fixture
fixture_object_count += objects_in_fixture
label_found = True
finally:
fixture.close()
# If the fixture we loaded contains 0 objects, assume that an
# error was encountered during fixture loading.
if objects_in_fixture == 0:
print("No fixture data found for '%s'. (File format may be invalid.)\n" %
(fixture_name))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(using=using)
return
# Since we disabled constraint checks, we must manually check for
# any invalid keys that might have been added
table_names = [model._meta.db_table for model in models]
def handle(self, *fixture_labels, **options):
""" Main method of a Django command """
from django.db.models import get_apps
from django.core import serializers
from django.conf import settings
self.style = no_style()
verbosity = int(options.get('verbosity', 1))
show_traceback = options.get('traceback', False)
# Keep a count of the installed objects and fixtures
fixture_count = 0
object_count = 0
objects_per_fixture = []
models = set()
humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
app_fixtures = [
os.path.join(os.path.dirname(app.__file__), 'fixtures')
for app in get_apps()
]
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) == 1:
fixture_name = fixture_label
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats():
formats = [format]
else:
formats = []
if formats:
if verbosity > 1:
print("Loading '%s' fixtures..." % fixture_name)
else:
sys.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s is not a known serialization format."
% (fixture_name, format)))
transaction.rollback()
return
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(
settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity > 1:
print("Checking %s for fixtures..." %
humanize(fixture_dir))
label_found = False
for format in formats:
if verbosity > 1:
print("Trying %s for %s fixture '%s'..." %
(humanize(fixture_dir), format, fixture_name))
try:
full_path = os.path.join(
fixture_dir, '.'.join([fixture_name, format]))
fixture = open(full_path, 'r')
if label_found:
fixture.close()
print(
self.style.ERROR(
"Multiple fixtures named '%s' in %s. Aborting."
% (fixture_name, humanize(fixture_dir))))
transaction.rollback()
return
else:
fixture_count += 1
objects_per_fixture.append(0)
if verbosity > 0:
print("Installing %s fixture '%s' from %s." %
(format, fixture_name,
humanize(fixture_dir)))
try:
objects_to_keep = {}
objects = serializers.deserialize(
format, fixture)
for obj in objects:
object_count += 1
objects_per_fixture[-1] += 1
class_ = obj.object.__class__
if class_ not in objects_to_keep:
objects_to_keep[class_] = set()
objects_to_keep[class_].add(obj.object)
models.add(class_)
obj.save()
if options.get('remove'):
self.remove_objects_not_in(
objects_to_keep, verbosity)
label_found = True
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
import traceback
fixture.close()
transaction.rollback()
if show_traceback:
traceback.print_exc()
else:
sys.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s\n"
% (full_path,
traceback.format_exc())))
return
fixture.close()
except:
if verbosity > 1:
print(
"No %s fixture '%s' in %s." %
(format, fixture_name, humanize(fixture_dir)))
# If any of the fixtures we loaded contain 0 objects, assume that an
# error was encountered during fixture loading.
if 0 in objects_per_fixture:
sys.stderr.write(
self.style.ERROR(
"No fixture data found for '%s'. (File format may be invalid.)"
% fixture_name))
transaction.rollback()
return
# If we found even one object in a fixture, we need to reset the
# database sequences.
if object_count > 0:
sequence_sql = connection.ops.sequence_reset_sql(
self.style, models)
if sequence_sql:
if verbosity > 1:
print("Resetting sequences")
for line in sequence_sql:
cursor.execute(line)
transaction.commit()
if object_count == 0:
if verbosity > 1:
print("No fixtures found.")
else:
if verbosity > 0:
print("Installed %d object(s) from %d fixture(s)" %
(object_count, fixture_count))
# Close the DB connection. This is required as a workaround for an
# edge case in MySQL: if the same connection is used to
# create tables, load data, and query, the query can return
# incorrect results. See Django #7572, MySQL #37735.
connection.close()
def handle(self, *args, **options):
for app in get_apps():
couchdbkit_handler.sync(app, verbosity=2)
def handle(self, *args, **options):
try:
migrations_dir = settings.DMIGRATIONS_DIR
except AttributeError:
print "You need to add DMIGRATIONS_DIR to your settings"
return
migration_db = MigrationDb(directory = migrations_dir)
migration_state = MigrationState(
migration_db = migration_db, dev = options.get('dev')
)
verbosity = int(options.get('verbosity', 1))
if not args or args[0] == 'help':
self.print_help(sys.argv[0], 'dmigrate')
return
elif args[0] in 'all all_hard up down upto downto to apply unapply'.split():
migration_state.init()
for (migration_name, action) in migration_state.plan(*args):
migration = migration_db.load_migration_object(migration_name)
start_time = time.time()
if action == 'up':
if verbosity >= 1:
print "Applying migration %s" % migration.name
if not options.get('print_plan'):
migration_state.apply(migration_name)
else:
if verbosity >= 1:
print "Unapplying migration %s" % migration.name
if not options.get('print_plan'):
migration_state.unapply(migration_name)
if options.get('print_time'):
print "Migration %s ran %.1f seconds" % (migration.name, time.time() - start_time)
elif args[0] == 'mark_as_applied':
migration_state.init()
for name in args[1:]:
resolved_name = migration_state.resolve_name(name)
if resolved_name == None:
raise NoSuchMigrationError(name)
migration_state.mark_as_applied(resolved_name)
elif args[0] == 'mark_as_unapplied':
migration_state.init()
for name in args[1:]:
resolved_name = migration_state.resolve_name(name)
if resolved_name == None:
raise NoSuchMigrationError(name)
migration_state.mark_as_unapplied(resolved_name)
elif args[0] == 'list':
migration_state.init()
for migration_name in migration_db.list():
if migration_state.is_applied(migration_name, use_cache=True):
print "* [+] %s" % migration_name
else:
print "* [ ] %s" % migration_name
migrations_not_in_db = migration_state.applied_but_not_in_db()
if migrations_not_in_db:
print "These migrations are marked as applied but cannot " \
"be found:"
for migration_name in migrations_not_in_db:
print "* [?] %s" % migration_name
return
elif args[0] == 'init':
migration_state.init()
elif args[0] == 'cat':
for name in args[1:]:
print open(
migration_db.resolve_migration_path(name), 'r'
).read()
return
else:
raise CommandError(
'Argument should be one of: list, help, up, down, all, all_hard, init, '
'apply, unapply, to, downto, upto, mark_as_applied, '
'mark_as_unapplied'
)
# Ensure Django permissions and content_types have been created
# NOTE: Don't run if django_content_type doesn't exist yet.
if table_present('django_content_type'):
from django.contrib.auth.management import create_permissions
from django.db import models
for app in models.get_apps():
if verbosity >= 1:
create_permissions(app, set(), 2)
else:
create_permissions(app, set(), 1)
def get_modelclasses():
for app in get_apps():
modelclasses = get_models(app)
for modelclass in modelclasses:
yield modelclass
def update_all_contenttypes(verbosity=2):
for app in get_apps():
update_contenttypes(app, None, verbosity)
def run_tests(test_labels,
verbosity=1,
interactive=True,
extra_tests=[],
suite=None):
"""
This module allows users to run tests for GIS apps that require the creation
of a spatial database. Currently, this is only required for PostgreSQL as
PostGIS needs extra overhead in test database creation.
In order to create a PostGIS database, the DATABASE_USER (or
TEST_DATABASE_USER, if defined) will require superuser priviliges.
To accomplish this outside the `postgres` user, you have a few options:
(A) Make your user a super user:
This may be done at the time the user is created, for example:
$ createuser --superuser <user_name>
Or you may alter the user's role from the SQL shell (assuming this
is done from an existing superuser role):
postgres# ALTER ROLE <user_name> SUPERUSER;
(B) Create your own PostgreSQL database as a local user:
1. Initialize database: `initdb -D /path/to/user/db`
2. If there's already a Postgres instance on the machine, it will need
to use a different TCP port than 5432. Edit postgresql.conf (in
/path/to/user/db) to change the database port (e.g. `port = 5433`).
3. Start this database `pg_ctl -D /path/to/user/db start`
(C) On Windows platforms the pgAdmin III utility may also be used as
a simple way to add superuser privileges to your database user.
The TEST_RUNNER needs to be set in your settings like so:
TEST_RUNNER='django.contrib.gis.tests.run_tests'
Note: This test runner assumes that the PostGIS SQL files ('lwpostgis.sql'
and 'spatial_ref_sys.sql') are installed in the directory specified by
`pg_config --sharedir` (and defaults to /usr/local/share if that fails).
This behavior is overridden if POSTGIS_SQL_PATH is set in your settings.
Windows users should set POSTGIS_SQL_PATH manually because the output
of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'.
Finally, the tests may be run by invoking `./manage.py test`.
"""
# The `create_spatial_db` routine abstracts away all the steps needed
# to properly construct a spatial database for the backend.
from django.contrib.gis.db.backend import create_spatial_db
# Setting up for testing.
setup_test_environment()
settings.DEBUG = False
old_name = settings.DATABASE_NAME
# The suite may be passed in manually, e.g., when we run the GeoDjango test,
# we want to build it and pass it in due to some customizations. Otherwise,
# the normal test suite creation process from `django.test.simple.run_tests`
# is used to create the test suite.
if suite is None:
suite = unittest.TestSuite()
if test_labels:
for label in test_labels:
if '.' in label:
suite.addTest(build_test(label))
else:
app = get_app(label)
suite.addTest(build_suite(app))
else:
for app in get_apps():
suite.addTest(build_suite(app))
for test in extra_tests:
suite.addTest(test)
# Creating the test spatial database.
create_spatial_db(test=True, verbosity=verbosity)
# Executing the tests (including the model tests), and destorying the
# test database after the tests have completed.
result = unittest.TextTestRunner(verbosity=verbosity).run(suite)
connection.creation.destroy_test_db(old_name, verbosity)
teardown_test_environment()
# Returning the total failures and errors
return len(result.failures) + len(result.errors)
def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]):
"""
worsk exactly as per normal test
but only creates the test_db if it doesn't yet exist
and does not destroy it when done
tables are flushed and fixtures loaded between tests as per usual
but if your schema has not changed then this saves significant amounts of time
and speeds up the test cycle
Run the unit tests for all the test labels in the provided list.
Labels must be of the form:
- app.TestClass.test_method
Run a single specific test method
- app.TestClass
Run all the test methods in a given class
- app
Search for doctests and unittests in the named application.
When looking for tests, the test runner will look in the models and
tests modules for the application.
A list of 'extra' tests may also be provided; these tests
will be added to the test suite.
Returns the number of tests that failed.
"""
setup_test_environment()
settings.DEBUG = False
suite = unittest.TestSuite()
if test_labels:
for label in test_labels:
if '.' in label:
suite.addTest(build_test(label))
else:
app = get_app(label)
suite.addTest(build_suite(app))
else:
for app in get_apps():
suite.addTest(build_suite(app))
for test in extra_tests:
suite.addTest(test)
suite = reorder_suite(suite, (TestCase, ))
old_name = settings.DATABASES['default']['NAME']
###Everything up to here is from django.test.simple
from django.db.backends import creation
from django.db import connection, DatabaseError
if settings.DATABASES['default']['TEST_NAME']:
settings.DATABASES['default']['NAME'] = settings.DATABASES['default'][
'TEST_NAME']
else:
settings.DATABASES['default'][
'NAME'] = creation.TEST_DATABASE_PREFIX + settings.DATABASES[
'default']['NAME']
connection.settings_dict["DATABASE_NAME"] = settings.DATABASES['default'][
'NAME']
# does test db exist already ?
try:
if settings.DATABASES['default']['ENGINE'] == 'sqlite3':
if not os.path.exists(settings.DATABASES['default']['NAME']):
raise DatabaseError
cursor = connection.cursor()
except Exception:
# db does not exist
# juggling ! create_test_db switches the DATABASE_NAME to the TEST_DATABASE_NAME
settings.DATABASES['default']['NAME'] = old_name
connection.settings_dict["DATABASE_NAME"] = old_name
connection.creation.create_test_db(verbosity, autoclobber=True)
else:
connection.close()
settings.DATABASES['default'][
'SUPPORTS_TRANSACTIONS'] = connections_support_transactions()
result = unittest.TextTestRunner(verbosity=verbosity).run(suite)
#Since we don't call destory_test_db, we need to set the db name back.
settings.DATABASES['default']['NAME'] = old_name
connection.settings_dict["DATABASE_NAME"] = old_name
teardown_test_environment()
return len(result.failures) + len(result.errors)
def sync_apps(self, connection, apps):
"Runs the old syncdb-style operation on a list of apps."
cursor = connection.cursor()
# Get a list of already installed *models* so that references work right.
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
created_models = set()
pending_references = {}
# Build the manifest of apps and models that are to be synchronized
all_models = [(app.__name__.split('.')[-2],
router.get_migratable_models(app,
connection.alias,
include_auto_created=True))
for app in models.get_apps()
if app.__name__.split('.')[-2] in apps]
def model_installed(model):
opts = model._meta
converter = connection.introspection.table_name_converter
# Note that if a model is unmanaged we short-circuit and never try to install it
return not ((converter(opts.db_table) in tables) or
(opts.auto_created and converter(
opts.auto_created._meta.db_table) in tables))
manifest = OrderedDict(
(app_name, list(filter(model_installed, model_list)))
for app_name, model_list in all_models)
create_models = set(itertools.chain(*manifest.values()))
emit_pre_migrate_signal(create_models, self.verbosity,
self.interactive, connection.alias)
# Create the tables for each model
if self.verbosity >= 1:
self.stdout.write(" Creating tables...\n")
with transaction.atomic(using=connection.alias, savepoint=False):
for app_name, model_list in manifest.items():
for model in model_list:
# Create the model's database table, if it doesn't already exist.
if self.verbosity >= 3:
self.stdout.write(" Processing %s.%s model\n" %
(app_name, model._meta.object_name))
sql, references = connection.creation.sql_create_model(
model, no_style(), seen_models)
seen_models.add(model)
created_models.add(model)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(
connection.creation.sql_for_pending_references(
refto, no_style(), pending_references))
sql.extend(
connection.creation.sql_for_pending_references(
model, no_style(), pending_references))
if self.verbosity >= 1 and sql:
self.stdout.write(" Creating table %s\n" %
model._meta.db_table)
for statement in sql:
cursor.execute(statement)
tables.append(
connection.introspection.table_name_converter(
model._meta.db_table))
# We force a commit here, as that was the previous behaviour.
# If you can prove we don't need this, remove it.
transaction.set_dirty(using=connection.alias)
# The connection may have been closed by a syncdb handler.
cursor = connection.cursor()
# Install custom SQL for the app (but only if this
# is a model we've just created)
if self.verbosity >= 1:
self.stdout.write(" Installing custom SQL...\n")
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
custom_sql = custom_sql_for_model(model, no_style(),
connection)
if custom_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
with transaction.commit_on_success_unless_managed(
using=connection.alias):
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install custom SQL for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e))
if self.show_traceback:
traceback.print_exc()
else:
if self.verbosity >= 3:
self.stdout.write(
" No custom SQL for %s.%s model\n" %
(app_name, model._meta.object_name))
if self.verbosity >= 1:
self.stdout.write(" Installing indexes...\n")
# Install SQL indices for all newly created models
for app_name, model_list in manifest.items():
for model in model_list:
if model in created_models:
index_sql = connection.creation.sql_indexes_for_model(
model, no_style())
if index_sql:
if self.verbosity >= 2:
self.stdout.write(
" Installing index for %s.%s model\n" %
(app_name, model._meta.object_name))
try:
with transaction.commit_on_success_unless_managed(
using=connection.alias):
for sql in index_sql:
cursor.execute(sql)
except Exception as e:
self.stderr.write(
" Failed to install index for %s.%s model: %s\n"
% (app_name, model._meta.object_name, e))
# Load initial_data fixtures (unless that has been disabled)
if self.load_initial_data:
call_command('loaddata',
'initial_data',
verbosity=self.verbosity,
database=connection.alias,
skip_validation=True)
return created_models
#!/usr/bin/env python
from django.core.management import setup_environ
try:
import settings
except ImportError:
import sys
sys.stderr.write("Couldn't find the settings.py module.")
sys.exit(1)
setup_environ(settings)
# Add any missing content types
from django.contrib.contenttypes.management import update_all_contenttypes
update_all_contenttypes()
# Add any missing permissions
from django.contrib.auth.management import create_permissions
from django.db.models import get_apps
for app in get_apps():
create_permissions(app, None, 2)
print('done!')
def handle(self, *fixture_labels, **options):
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
self.style = no_style()
verbosity = int(options.get('verbosity', 1))
show_traceback = options.get('traceback', False)
# commit is a stealth option - it isn't really useful as
# a command line option, but it can be useful when invoking
# loaddata from within another script.
# If commit=True, loaddata will use its own transaction;
# if commit=False, the data load SQL will become part of
# the transaction in place when loaddata was invoked.
commit = options.get('commit', True)
# Keep a count of the installed objects and fixtures
fixture_count = 0
loaded_object_count = 0
fixture_object_count = 0
models = set()
humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
if commit:
transaction.commit_unless_managed(using=using)
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
class SingleZipReader(zipfile.ZipFile):
def __init__(self, *args, **kwargs):
zipfile.ZipFile.__init__(self, *args, **kwargs)
if settings.DEBUG:
assert len(
self.namelist()
) == 1, "Zip-compressed fixtures must contain only one file."
def read(self):
return zipfile.ZipFile.read(self, self.namelist()[0])
compression_types = {
None: open,
'gz': gzip.GzipFile,
'zip': SingleZipReader
}
if has_bz2:
compression_types['bz2'] = bz2.BZ2File
app_module_paths = []
for app in get_apps():
if hasattr(app, '__path__'):
# It's a 'models/' subpackage
for path in app.__path__:
app_module_paths.append(path)
else:
# It's a models.py module
app_module_paths.append(app.__file__)
app_fixtures = [
os.path.join(os.path.dirname(path), 'fixtures')
for path in app_module_paths
]
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) > 1 and parts[-1] in compression_types:
compression_formats = [parts[-1]]
parts = parts[:-1]
else:
compression_formats = compression_types.keys()
if len(parts) == 1:
fixture_name = parts[0]
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats():
formats = [format]
else:
formats = []
if formats:
if verbosity >= 2:
self.stdout.write("Loading '%s' fixtures...\n" %
fixture_name)
else:
self.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s is not a known serialization format.\n"
% (fixture_name, format)))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(using=using)
return
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(
settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity >= 2:
self.stdout.write("Checking %s for fixtures...\n" %
humanize(fixture_dir))
label_found = False
for combo in product([using, None], formats,
compression_formats):
database, format, compression_format = combo
file_name = '.'.join(
p for p in
[fixture_name, database, format, compression_format]
if p)
if verbosity >= 3:
self.stdout.write("Trying %s for %s fixture '%s'...\n" % \
(humanize(fixture_dir), file_name, fixture_name))
full_path = os.path.join(fixture_dir, file_name)
open_method = compression_types[compression_format]
try:
fixture = open_method(full_path, 'r')
if label_found:
fixture.close()
self.stderr.write(
self.style.ERROR(
"Multiple fixtures named '%s' in %s. Aborting.\n"
% (fixture_name, humanize(fixture_dir))))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(
using=using)
return
else:
fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
if verbosity >= 2:
self.stdout.write("Installing %s fixture '%s' from %s.\n" % \
(format, fixture_name, humanize(fixture_dir)))
try:
objects = serializers.deserialize(format,
fixture,
using=using)
with connection.constraint_checks_disabled():
for obj in objects:
objects_in_fixture += 1
if router.allow_syncdb(
using, obj.object.__class__):
loaded_objects_in_fixture += 1
models.add(obj.object.__class__)
obj.save(using=using)
# Since we disabled constraint checks, we must manually check for
# any invalid keys that might have been added
table_names = [
model._meta.db_table for model in models
]
connection.check_constraints(
table_names=table_names)
loaded_object_count += loaded_objects_in_fixture
fixture_object_count += objects_in_fixture
label_found = True
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
import traceback
fixture.close()
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(
using=using)
if show_traceback:
traceback.print_exc()
else:
self.stderr.write(
self.style.ERROR(
"Problem installing fixture '%s': %s\n"
% (full_path, ''.join(
traceback.format_exception(
sys.exc_type,
sys.exc_value,
sys.exc_traceback)))))
return
fixture.close()
# If the fixture we loaded contains 0 objects, assume that an
# error was encountered during fixture loading.
if objects_in_fixture == 0:
self.stderr.write(
self.style.ERROR(
"No fixture data found for '%s'. (File format may be invalid.)\n"
% (fixture_name)))
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(
using=using)
return
except Exception, e:
if verbosity >= 2:
self.stdout.write("No %s fixture '%s' in %s.\n" % \
(format, fixture_name, humanize(fixture_dir)))
def handle_noargs(self, **options):
from django.contrib.auth.management import create_permissions
from django.db.models import get_apps
for app in get_apps():
create_permissions(app, None, 2)
def model_from_db_table(db_table):
for app in models.get_apps():
for model in models.get_models(app):
if model._meta.db_table == db_table:
return model
raise ValueError("Couldn't find model class for %s" % db_table)
def generate_dot(app_labels, **kwargs):
disable_fields = kwargs.get('disable_fields', False)
include_models = parse_file_or_list(kwargs.get('include_models', ""))
all_applications = kwargs.get('all_applications', False)
use_subgraph = kwargs.get('group_models', False)
verbose_names = kwargs.get('verbose_names', False)
inheritance = kwargs.get('inheritance', False)
language = kwargs.get('language', None)
if language is not None:
activate_language(language)
exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', ""))
exclude_models = parse_file_or_list(kwargs.get('exclude_models', ""))
def skip_field(field):
if exclude_columns:
if verbose_names and field.verbose_name:
if field.verbose_name in exclude_columns:
return True
if field.name in exclude_columns:
return True
return False
t = loader.get_template_from_string("""
digraph name {
fontname = "Helvetica"
fontsize = 8
node [
fontname = "Helvetica"
fontsize = 8
shape = "plaintext"
]
edge [
fontname = "Helvetica"
fontsize = 8
]
""")
c = Context({})
dot = t.render(c)
apps = []
if all_applications:
apps = models.get_apps()
for app_label in app_labels:
app = models.get_app(app_label)
if not app in apps:
apps.append(app)
graphs = []
for app in apps:
graph = Context({
'name':
'"%s"' % app.__name__,
'app_name':
"%s" % '.'.join(app.__name__.split('.')[:-1]),
'cluster_app_name':
"cluster_%s" % app.__name__.replace(".", "_"),
'disable_fields':
disable_fields,
'use_subgraph':
use_subgraph,
'models': []
})
appmodels = get_models(app)
abstract_models = []
for appmodel in appmodels:
abstract_models = abstract_models + [
abstract_model
for abstract_model in appmodel.__bases__ if hasattr(
abstract_model, '_meta') and abstract_model._meta.abstract
]
abstract_models = list(set(abstract_models)) # remove duplicates
appmodels = abstract_models + appmodels
for appmodel in appmodels:
appmodel_abstracts = [
abstract_model.__name__
for abstract_model in appmodel.__bases__
if hasattr(abstract_model, '_meta')
and abstract_model._meta.abstract
]
# collect all attribs of abstract superclasses
def getBasesAbstractFields(c):
_abstract_fields = []
for e in c.__bases__:
if hasattr(e, '_meta') and e._meta.abstract:
_abstract_fields.extend(e._meta.fields)
_abstract_fields.extend(getBasesAbstractFields(e))
return _abstract_fields
abstract_fields = getBasesAbstractFields(appmodel)
model = {
'app_name': appmodel.__module__.replace(".", "_"),
'name': appmodel.__name__,
'abstracts': appmodel_abstracts,
'fields': [],
'relations': []
}
# consider given model name ?
def consider(model_name):
if exclude_models and model_name in exclude_models:
return False
return not include_models or model_name in include_models
if not consider(appmodel._meta.object_name):
continue
if verbose_names and appmodel._meta.verbose_name:
model['label'] = appmodel._meta.verbose_name
else:
model['label'] = model['name']
# model attributes
def add_attributes(field):
if verbose_names and field.verbose_name:
label = field.verbose_name
else:
label = field.name
t = type(field).__name__
if isinstance(field, (OneToOneField, ForeignKey)):
t += " ({0})".format(field.rel.field_name)
# TODO: ManyToManyField, GenericRelation
model['fields'].append({
'name': field.name,
'label': label,
'type': t,
'blank': field.blank,
'abstract': field in abstract_fields,
})
# Find all the real attributes. Relations are depicted as graph edges instead of attributes
attributes = [
field for field in appmodel._meta.local_fields
if not isinstance(field, RelatedField)
]
# find primary key and print it first, ignoring implicit id if other pk exists
pk = appmodel._meta.pk
if not appmodel._meta.abstract and pk in attributes:
add_attributes(pk)
for field in attributes:
if skip_field(field):
continue
if not field.primary_key:
add_attributes(field)
# FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph?
#if appmodel._meta.many_to_many:
# for field in appmodel._meta.many_to_many:
# if skip_field(field):
# continue
# add_attributes(field)
# relations
def add_relation(field, extras=""):
if verbose_names and field.verbose_name:
label = field.verbose_name
else:
label = field.name
# show related field name
if hasattr(field, 'related_query_name'):
label += ' (%s)' % field.related_query_name()
# handle self-relationships
if field.rel.to == 'self':
target_model = field.model
else:
target_model = field.rel.to
_rel = {
'target_app': target_model.__module__.replace('.', '_'),
'target': target_model.__name__,
'type': type(field).__name__,
'name': field.name,
'label': label,
'arrows': extras,
'needs_node': True
}
if _rel not in model['relations'] and consider(_rel['target']):
model['relations'].append(_rel)
for field in appmodel._meta.local_fields:
if field.attname.endswith(
'_ptr_id'
): # excluding field redundant with inheritance relation
continue
if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields
continue
if skip_field(field):
continue
if isinstance(field, OneToOneField):
add_relation(field, '[arrowhead=none, arrowtail=none]')
elif isinstance(field, ForeignKey):
add_relation(field, '[arrowhead=none, arrowtail=dot]')
for field in appmodel._meta.local_many_to_many:
if skip_field(field):
continue
if isinstance(field, ManyToManyField):
if (getattr(field, 'creates_table', False)
or # django 1.1.
(hasattr(field.rel.through, '_meta') and
field.rel.through._meta.auto_created)): # django 1.2
add_relation(
field, '[arrowhead=dot arrowtail=dot, dir=both]')
elif isinstance(field, GenericRelation):
add_relation(
field,
mark_safe(
'[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]'
))
if inheritance:
# add inheritance arrows
for parent in appmodel.__bases__:
if hasattr(parent, "_meta"): # parent is a model
l = "multi-table"
if parent._meta.abstract:
l = "abstract"
if appmodel._meta.proxy:
l = "proxy"
l += r"\ninheritance"
_rel = {
'target_app': parent.__module__.replace(".", "_"),
'target': parent.__name__,
'type': "inheritance",
'name': "inheritance",
'label': l,
'arrows': '[arrowhead=empty, arrowtail=none]',
'needs_node': True
}
# TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes.
if _rel not in model['relations'] and consider(
_rel['target']):
model['relations'].append(_rel)
graph['models'].append(model)
graphs.append(graph)
nodes = []
for graph in graphs:
nodes.extend([e['name'] for e in graph['models']])
for graph in graphs:
# don't draw duplication nodes because of relations
for model in graph['models']:
for relation in model['relations']:
if relation['target'] in nodes:
relation['needs_node'] = False
# render templates
t = loader.get_template_from_string("""{% if use_subgraph %}
subgraph {{ cluster_app_name }} {
label=<
<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0">
<TR><TD COLSPAN="2" CELLPADDING="4" ALIGN="CENTER"
><FONT FACE="Helvetica Bold" COLOR="Black" POINT-SIZE="12"
>{{ app_name }}</FONT></TD></TR>
</TABLE>
>
color=olivedrab4
style="rounded"
{% endif %}
{% for model in models %}
{{ model.app_name }}_{{ model.name }} [label=<
<TABLE BGCOLOR="palegoldenrod" BORDER="0" CELLBORDER="0" CELLSPACING="0">
<TR><TD COLSPAN="2" CELLPADDING="4" ALIGN="CENTER" BGCOLOR="olivedrab4"
><FONT FACE="Helvetica Bold" COLOR="white"
>{{ model.label }}{% if model.abstracts %}<BR/><<FONT FACE="Helvetica Italic">{{ model.abstracts|join:"," }}</FONT>>{% endif %}</FONT></TD></TR>
{% if not disable_fields %}
{% for field in model.fields %}
<TR><TD ALIGN="LEFT" BORDER="0"
><FONT {% if field.blank %}COLOR="#7B7B7B" {% endif %}FACE="Helvetica {% if field.abstract %}Italic{% else %}Bold{% endif %}">{{ field.label }}</FONT
></TD>
<TD ALIGN="LEFT"
><FONT {% if field.blank %}COLOR="#7B7B7B" {% endif %}FACE="Helvetica {% if field.abstract %}Italic{% else %}Bold{% endif %}">{{ field.type }}</FONT
></TD></TR>
{% endfor %}
{% endif %}
</TABLE>
>]
{% endfor %}
{% if use_subgraph %}
}
{% endif %}""")
dot += '\n' + t.render(graph)
for graph in graphs:
t = loader.get_template_from_string("""{% for model in models %}
{% for relation in model.relations %}
{% if relation.needs_node %}
{{ relation.target_app }}_{{ relation.target }} [label=<
<TABLE BGCOLOR="palegoldenrod" BORDER="0" CELLBORDER="0" CELLSPACING="0">
<TR><TD COLSPAN="2" CELLPADDING="4" ALIGN="CENTER" BGCOLOR="olivedrab4"
><FONT FACE="Helvetica Bold" COLOR="white"
>{{ relation.target }}</FONT></TD></TR>
</TABLE>
>]
{% endif %}
{{ model.app_name }}_{{ model.name }} -> {{ relation.target_app }}_{{ relation.target }}
[label="{{ relation.label }}"] {{ relation.arrows }};
{% endfor %}
{% endfor %}""")
dot += '\n' + t.render(graph)
t = loader.get_template_from_string("}")
c = Context({})
dot += '\n' + t.render(c)
return dot
def load_initial_objects(self):
# look for any .xml files in apps under fixtures/initial_objects
# and attempt to load them as Fedora objects
# NOTE! any fixtures should have pids specified, or new versions of the
# fixture will be created every time syncrepo runs
app_module_paths = []
if hasattr(django_apps, 'get_app_configs'):
apps = django_apps.get_app_configs()
else:
apps = get_apps()
# monkey see django code, monkey do
for app in apps:
# newer django AppConfig
if hasattr(app, 'path'):
app_module_paths.append(app.path)
elif hasattr(app, '__path__'):
# It's a 'models/' subpackage
for path in app.__path__:
app_module_paths.append(path)
else:
# It's a models.py module
app_module_paths.append(app.__file__)
app_fixture_paths = [
os.path.join(os.path.dirname(path), 'fixtures', 'initial_objects',
'*.xml') for path in app_module_paths
]
fixture_count = 0
load_count = 0
for path in app_fixture_paths:
fixtures = glob.iglob(path)
for f in fixtures:
# FIXME: is there a sane, sensible way to shorten file path for error/success messages?
fixture_count += 1
with open(f) as fixture_data:
# rather than pulling PID from fixture and checking if it already exists,
# just ingest and catch appropriate excetions
try:
pid = self.repo.ingest(fixture_data.read(),
"loaded from fixture")
if self.verbosity > 1:
self.stdout.write("Loaded fixture %s as %s" %
(f, pid))
load_count += 1
except RequestFailed as rf:
if hasattr(rf, 'detail'):
if 'ObjectExistsException' in rf.detail or \
'already exists in the registry; the object can\'t be re-created' in rf.detail:
if self.verbosity > 1:
self.stdout.write(
"Fixture %s has already been loaded" %
f)
elif 'ObjectValidityException' in rf.detail:
# could also look for: fedora.server.errors.ValidationException
# (e.g., RELS-EXT about does not match pid)
self.stdout.write(
"Error: fixture %s is not a valid Repository object"
% f)
else:
# if there is at least a detail message, display that
self.stdout.write("Error ingesting %s: %s" %
(f, rf.detail))
else:
raise rf
# summarize what was actually done
if self.verbosity > 0:
if fixture_count == 0:
self.stdout.write("No fixtures found")
else:
self.stdout.write("Loaded %d object(s) from %d fixture(s)" %
(load_count, fixture_count))
def Deserializer(object_list, **options):
"""
Deserialize simple Python objects back into Django ORM instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
db = options.pop('using', DEFAULT_DB_ALIAS)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value,
options.get(
"encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
if hasattr(field.rel.to._default_manager,
'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__'):
return field.rel.to._default_manager.db_manager(
db).get_by_natural_key(*value).pk
else:
return smart_unicode(
field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_unicode(
field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
if hasattr(field.rel.to._default_manager,
'get_by_natural_key'):
if hasattr(field_value, '__iter__'):
obj = field.rel.to._default_manager.db_manager(
db).get_by_natural_key(*field_value)
value = getattr(obj, field.rel.field_name)
# If this is a natural foreign key to an object that
# has a FK/O2O as the foreign key, use the FK value
if field.rel.to._meta.pk.rel:
value = value.pk
else:
value = field.rel.to._meta.get_field(
field.rel.field_name).to_python(field_value)
data[field.attname] = value
else:
data[field.attname] = field.rel.to._meta.get_field(
field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def handle_noargs(self, **options):
db = options.get('database')
connection = connections[db]
verbosity = int(options.get('verbosity'))
interactive = options.get('interactive')
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError:
pass
sql_list = sql_flush(self.style, connection, only_django=True)
if interactive:
confirm = raw_input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to the state it was in after syncdb.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ %
connection.settings_dict['NAME'])
else:
confirm = 'yes'
if confirm == 'yes':
try:
cursor = connection.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception as e:
transaction.rollback_unless_managed(using=db)
raise CommandError(
"""Database %s couldn't be flushed. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the expected database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
The full error: %s""" % (connection.settings_dict['NAME'], e))
transaction.commit_unless_managed(using=db)
# Emit the post sync signal. This allows individual
# applications to respond as if the database had been
# sync'd from scratch.
all_models = []
for app in models.get_apps():
all_models.extend([
m
for m in models.get_models(app, include_auto_created=True)
if router.allow_syncdb(db, m)
])
emit_post_sync_signal(set(all_models), verbosity, interactive, db)
# Reinstall the initial_data fixture.
kwargs = options.copy()
kwargs['database'] = db
if options.get('load_initial_data', True):
# Reinstall the initial_data fixture.
from django.core.management import call_command
call_command('loaddata', 'initial_data', **options)
else:
self.stdout.write("Flush cancelled.\n")
def handle(self, *fixture_labels, **options):
using = options.get('database')
connection = connections[using]
if not len(fixture_labels):
raise CommandError(
"No database fixture specified. Please provide the path of at "
"least one fixture in the command line.")
verbosity = int(options.get('verbosity'))
show_traceback = options.get('traceback')
# commit is a stealth option - it isn't really useful as
# a command line option, but it can be useful when invoking
# loaddata from within another script.
# If commit=True, loaddata will use its own transaction;
# if commit=False, the data load SQL will become part of
# the transaction in place when loaddata was invoked.
commit = options.get('commit', True)
# Keep a count of the installed objects and fixtures
fixture_count = 0
loaded_object_count = 0
fixture_object_count = 0
models = set()
humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path'
# Get a cursor (even though we don't need one yet). This has
# the side effect of initializing the test database (if
# it isn't already initialized).
cursor = connection.cursor()
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
if commit:
transaction.commit_unless_managed(using=using)
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
class SingleZipReader(zipfile.ZipFile):
def __init__(self, *args, **kwargs):
zipfile.ZipFile.__init__(self, *args, **kwargs)
if settings.DEBUG:
assert len(
self.namelist()
) == 1, "Zip-compressed fixtures must contain only one file."
def read(self):
return zipfile.ZipFile.read(self, self.namelist()[0])
compression_types = {
None: open,
'gz': gzip.GzipFile,
'zip': SingleZipReader
}
if has_bz2:
compression_types['bz2'] = bz2.BZ2File
app_module_paths = []
for app in get_apps():
if hasattr(app, '__path__'):
# It's a 'models/' subpackage
for path in app.__path__:
app_module_paths.append(path)
else:
# It's a models.py module
app_module_paths.append(app.__file__)
app_fixtures = [
os.path.join(os.path.dirname(path), 'fixtures')
for path in app_module_paths
]
try:
with connection.constraint_checks_disabled():
for fixture_label in fixture_labels:
parts = fixture_label.split('.')
if len(parts) > 1 and parts[-1] in compression_types:
compression_formats = [parts[-1]]
parts = parts[:-1]
else:
compression_formats = compression_types.keys()
if len(parts) == 1:
fixture_name = parts[0]
formats = serializers.get_public_serializer_formats()
else:
fixture_name, format = '.'.join(parts[:-1]), parts[-1]
if format in serializers.get_public_serializer_formats(
):
formats = [format]
else:
formats = []
if formats:
if verbosity >= 2:
self.stdout.write("Loading '%s' fixtures..." %
fixture_name)
else:
raise CommandError(
"Problem installing fixture '%s': %s is not a known serialization format."
% (fixture_name, format))
if os.path.isabs(fixture_name):
fixture_dirs = [fixture_name]
else:
fixture_dirs = app_fixtures + list(
settings.FIXTURE_DIRS) + ['']
for fixture_dir in fixture_dirs:
if verbosity >= 2:
self.stdout.write("Checking %s for fixtures..." %
humanize(fixture_dir))
label_found = False
for combo in product([using, None], formats,
compression_formats):
database, format, compression_format = combo
file_name = '.'.join(p for p in [
fixture_name, database, format,
compression_format
] if p)
if verbosity >= 3:
self.stdout.write("Trying %s for %s fixture '%s'..." % \
(humanize(fixture_dir), file_name, fixture_name))
full_path = os.path.join(fixture_dir, file_name)
open_method = compression_types[compression_format]
try:
fixture = open_method(full_path, 'r')
except IOError:
if verbosity >= 2:
self.stdout.write("No %s fixture '%s' in %s." % \
(format, fixture_name, humanize(fixture_dir)))
else:
try:
if label_found:
raise CommandError(
"Multiple fixtures named '%s' in %s. Aborting."
% (fixture_name,
humanize(fixture_dir)))
fixture_count += 1
objects_in_fixture = 0
loaded_objects_in_fixture = 0
if verbosity >= 2:
self.stdout.write("Installing %s fixture '%s' from %s." % \
(format, fixture_name, humanize(fixture_dir)))
objects = serializers.deserialize(
format, fixture, using=using)
for obj in objects:
objects_in_fixture += 1
if router.allow_syncdb(
using, obj.object.__class__):
loaded_objects_in_fixture += 1
models.add(obj.object.__class__)
try:
obj.save(using=using)
except (DatabaseError,
IntegrityError) as e:
e.args = (
"Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s"
% {
'app_label':
obj.object._meta.
app_label,
'object_name':
obj.object._meta.
object_name,
'pk':
obj.object.pk,
'error_msg':
force_text(e)
}, )
raise
loaded_object_count += loaded_objects_in_fixture
fixture_object_count += objects_in_fixture
label_found = True
except Exception as e:
if not isinstance(e, CommandError):
e.args = (
"Problem installing fixture '%s': %s"
% (full_path, e), )
raise
finally:
fixture.close()
# If the fixture we loaded contains 0 objects, assume that an
# error was encountered during fixture loading.
if objects_in_fixture == 0:
raise CommandError(
"No fixture data found for '%s'. (File format may be invalid.)"
% (fixture_name))
# Since we disabled constraint checks, we must manually check for
# any invalid keys that might have been added
table_names = [model._meta.db_table for model in models]
try:
connection.check_constraints(table_names=table_names)
except Exception as e:
e.args = ("Problem installing fixtures: %s" % e, )
raise
except (SystemExit, KeyboardInterrupt):
raise
except Exception as e:
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(using=using)
raise
# If we found even one object in a fixture, we need to reset the
# database sequences.
if loaded_object_count > 0:
sequence_sql = connection.ops.sequence_reset_sql(
no_style(), models)
if sequence_sql:
if verbosity >= 2:
self.stdout.write("Resetting sequences\n")
for line in sequence_sql:
cursor.execute(line)
if commit:
transaction.commit(using=using)
transaction.leave_transaction_management(using=using)
if verbosity >= 1:
if fixture_object_count == loaded_object_count:
self.stdout.write("Installed %d object(s) from %d fixture(s)" %
(loaded_object_count, fixture_count))
else:
self.stdout.write(
"Installed %d object(s) (of %d) from %d fixture(s)" %
(loaded_object_count, fixture_object_count, fixture_count))
# Close the DB connection. This is required as a workaround for an
# edge case in MySQL: if the same connection is used to
# create tables, load data, and query, the query can return
# incorrect results. See Django #7572, MySQL #37735.
if commit:
connection.close()
