def Deserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ db = options.pop('using', DEFAULT_DB_ALIAS) models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__'): return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk else: return smart_unicode(field.rel.to._meta.pk.to_python(value)) else: m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v)) m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): if hasattr(field_value, '__iter__'): obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value) value = getattr(obj, field.rel.field_name) # If this is a natural foreign key to an object that # has a FK/O2O as the foreign key, use the FK value if field.rel.to._meta.pk.rel: value = value.pk else: value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) data[field.attname] = value else: data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def validate(cls, model): """ Does basic PermissionHandler option validation. Calls custom validation classmethod in the end if it is provided in cls. The signature of the custom validation classmethod should be: def validate(cls, model). """ # Before we can introspect models, they need to be fully loaded so that # inter-relations are set up correctly. We force that here. models.get_apps() # validate model class if not isinstance(model, ModelBase): # this mean the model is not subclass of models.Model raise ValidationError('The model "%s" must be a subclass of ``models.Model``.' % model.__class__.__name__) # validate handler class if not issubclass(cls, PermissionHandler): raise ValidationError( 'The handler "%s" must be a subclass of ' "``fluidpermission.handlers.PermissionHandler``." % cls.__class__.__name__ ) # call custom validation classmethod if hasattr(cls, "validate"): cls.validate(model)
def validate(cls, model): """ Validates if model is well configured """ # Before we can introspect models, they need to be fully loaded so that # inter-relations are set up correctly. We force that here. models.get_apps() # Call each option validation validate_fields(cls, model)
def Deserializer(object_list, **options): """Deserialize simple Python objects back into Model instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = python._get_model(d["model"]) data = {} key = resolve_key(Model._meta.module_name, d["pk"]) if key.name(): data["key_name"] = key.name() parent = None if key.parent(): parent = FakeParent(key.parent()) m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode( field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model.properties()[field_name] if isinstance(field, db.Reference): # Resolve foreign key references. data[field.name] = resolve_key(Model._meta.module_name, field_value) else: # Handle converting strings to more specific formats. if isinstance(field_value, basestring): if isinstance(field, db.DateProperty): field_value = datetime.datetime.strptime( field_value, '%Y-%m-%d').date() elif isinstance(field, db.TimeProperty): field_value = parse_datetime_with_microseconds(field_value, '%H:%M:%S').time() elif isinstance(field, db.DateTimeProperty): field_value = parse_datetime_with_microseconds(field_value, '%Y-%m-%d %H:%M:%S') # Handle pyyaml datetime.time deserialization - it returns a datetime # instead of a time. if (isinstance(field_value, datetime.datetime) and isinstance(field, db.TimeProperty)): field_value = field_value.time() data[field.name] = field.validate(field_value) # Create the new model instance with all it's data, but no parent. object = Model(**data) # Now add the parent into the hidden attribute, bypassing the type checks # in the Model's __init__ routine. object._parent = parent # When the deserialized object is saved our replacement DeserializedObject # class will set object._parent to force the real parent model to be loaded # the first time it is referenced. yield base.DeserializedObject(object, m2m_data)
def __init__(self, stream_or_string, **options): models.get_apps() super(Deserializer, self).__init__(stream_or_string, **options) self.wb = xlrd.open_workbook(stream_or_string.name, formatting_info=True) self.row_index=-1 self.sheet_index=-1 self.sheet=None self.model=None self.do_stop_iteration=False
def validate(cls, model): """ Does basic ModelAdmin option validation. Calls custom validation classmethod in the end if it is provided in cls. The signature of the custom validation classmethod should be: def validate(cls, model). """ # Before we can introspect models, they need to be fully loaded so that # inter-relations are set up correctly. We force that here. models.get_apps() opts = model._meta validate_base(cls, model) # list_display if hasattr(cls, 'list_display'): check_isseq(cls, 'list_display', cls.list_display) for idx, field in enumerate(cls.list_display): if not callable(field): if not hasattr(cls, field): if not hasattr(model, field): try: opts.get_field(field) except models.FieldDoesNotExist: raise ImproperlyConfigured("%s.list_display[%d], %r is not a callable or an attribute of %r or found in the model %r." % (cls.__name__, idx, field, cls.__name__, model._meta.object_name)) else: # getattr(model, field) could be an X_RelatedObjectsDescriptor f = fetch_attr(cls, model, opts, "list_display[%d]" % idx, field) if isinstance(f, models.ManyToManyField): raise ImproperlyConfigured("'%s.list_display[%d]', '%s' is a ManyToManyField which is not supported." % (cls.__name__, idx, field)) # list_display_links if hasattr(cls, 'list_display_links'): check_isseq(cls, 'list_display_links', cls.list_display_links) for idx, field in enumerate(cls.list_display_links): fetch_attr(cls, model, opts, 'list_display_links[%d]' % idx, field) if field not in cls.list_display: raise ImproperlyConfigured("'%s.list_display_links[%d]'" "refers to '%s' which is not defined in 'list_display'." % (cls.__name__, idx, field)) # list_filter if hasattr(cls, 'list_filter'): check_isseq(cls, 'list_filter', cls.list_filter) for idx, fpath in enumerate(cls.list_filter): try: get_fields_from_path(model, fpath) except (NotRelationField, FieldDoesNotExist), e: raise ImproperlyConfigured( "'%s.list_filter[%d]' refers to '%s' which does not refer to a Field." % ( cls.__name__, idx, fpath ) )
def Deserializer_with_debugging(original_function, object_list, **options): from django.core.serializers.python import _get_model from django.db import DEFAULT_DB_ALIAS from django.utils.encoding import smart_unicode from django.conf import settings print "loading all: %s" % object_list db = options.pop('using', DEFAULT_DB_ALIAS) db_models.get_apps() for d in object_list: print "loading %s" % d # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, db_models.ManyToManyRel): print " field = %s" % field print " field.rel = %s" % field.rel print " field.rel.to = %s" % field.rel.to print " field.rel.to._default_manager = %s" % ( field.rel.to._default_manager) print " field.rel.to.objects = %s" % ( field.rel.to.objects) if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__'): return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk else: return smart_unicode(field.rel.to._meta.pk.to_python(value)) else: m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v)) m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] for i, pk in enumerate(field_value): print " %s: converted %s to %s" % (field.name, pk, m2m_data[field.name][i]) result = original_function(object_list, **options) print " result = %s" % result import traceback traceback.print_stack() return result
def Deserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) uses_dictionary_lookup = False data = {} m2m_data = {} if 'pk' in d: data[Model._meta.pk.attname] = Model._meta.pk.to_python(d["pk"]) else: uses_dictionary_lookup = True # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): m2m_convert = field.rel.to._meta.pk.to_python rel_values = [] for rel_value in field_value: try: rel_values.append(m2m_convert(smart_unicode(rel_value))) except ValidationError: rel_values.append(rel_value) m2m_data[field.name] = rel_values # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: try: data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) except ValidationError: data[field.attname] = field_value else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield utils.DeserializedObject(Model, data, m2m_data)
def __init__(self, stream_or_string, **options): """ Init this serializer given a stream or a string """ self.options = options if isinstance(stream_or_string, basestring): self.stream = StringIO(stream_or_string) else: self.stream = stream_or_string # hack to make sure that the models have all been loaded before # deserialization starts (otherwise subclass calls to get_model() # and friends might fail...) models.get_apps()
def get_app(app=None, verbosity=0): """Uses django.db.models.get_app and fuzzywuzzy to get the models module for a django app Retrieve an app module from an app name string, even if mispelled (uses fuzzywuzzy to find the best match) To get a list of all the apps use `get_app(None)` or `get_app([]) or get_app(())` To get a single random app use `get_app('')` >>> get_app('call').__class__.__name__ == 'module' True >>> get_app('model').__name__ == 'miner.models' True >>> isinstance(get_app('whatever'), ModuleType) True >>> isinstance(get_app(''), ModuleType) True isinstance(get_app(), ModuleType) False isinstance(get_app(), list) True """ # print 'get_app(', app if not app: if not isinstance(app, (type(None), list, tuple)): if get_app.default: return get_app(get_app.default) else: return models.get_apps()[-1] else: return [app_class.__package__ for app_class in models.get_apps() if app_class and app_class.__package__] if isinstance(app, basestring) and app.strip().endswith('.models'): return get_app(app[:-len('.models')]) if isinstance(app, ModuleType): return app # print 'type(' + repr(app) + ') = ' + repr(type(app)) try: if verbosity > 1: print 'Attempting django.models.get_app(%r)' % app return models.get_app(app) except: if not app: if verbosity: print 'WARNING: app = %r, so returning None!' % app return None if verbosity > 2: print 'Trying a fuzzy match on app = %r' % app app_names = [app_class.__package__ for app_class in models.get_apps() if app_class and app_class.__package__] fuzzy_app_name = fuzzy.extractOne(str(app), app_names)[0] if verbosity: print 'WARNING: Best fuzzy match for app name %r is %s' % (app, fuzzy_app_name) return get_app(fuzzy_app_name)
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ if isinstance(stream_or_string, basestring): stream = StringIO(stream_or_string) else: stream = stream_or_string object_list = simplejson.load(stream) models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): m2m_convert = field.rel.to._meta.pk.to_python m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: # handle a dictionary which means to lookup the instance # and get the primary key this way (works great on # GFK values) if isinstance(field_value, dict): lookup_params = {} for k, v in field_value.iteritems(): lookup_params[k.encode("ascii")] = v field_value = field.rel.to._default_manager.get(**lookup_params).pk data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of YAML data. ******** All this is copied from the python base deserializer but for 2 lines ******** """ if isinstance(stream_or_string, basestring): stream = StringIO(stream_or_string) else: stream = stream_or_string object_list = yaml.load(stream) models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): m2m_convert = field.rel.to._meta.pk.to_python m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: #These are those 2 lines ****** if isinstance(field_value, dict): field_value = field.rel.to._default_manager.get(**field_value).pk data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def build_suite(self, test_labels, extra_tests=None, **kwargs): suite = unittest.TestSuite() if test_labels: for label in test_labels: if "." in label: from django.test.simple import build_test suite.addTest(build_test(label)) else: sub_suite = self.find_tests_and_apps(label) suite.addTest(sub_suite) else: from django.db.models import get_apps for app in get_apps(): from django.test.simple import build_suite suite.addTest(build_suite(app)) if extra_tests: for test in extra_tests: suite.addTest(test) from django.test.simple import reorder_suite from unittest import TestCase return reorder_suite(suite, (TestCase,))
def handle(self, *app_labels, **options): from django.db.models import get_app, get_apps exclude = options.get('exclude', []) excluded_apps = [get_app(app_label) for app_label in exclude] if len(app_labels) == 0: app_list = [app for app in get_apps() if app not in excluded_apps] else: app_list = [get_app(app_label) for app_label in app_labels] for app in app_list: name = app.__name__.replace('.models', '') app_path = __import__(name, {}, {}, [name.split('.')[-1]]).__path__ try: params = imp.find_module('generator', app_path) except ImportError: pass else: generator = imp.load_module(name + '.generator', *params) print "Generating items for %s." % name.title() start = time.time() load_json(generator.generate()) now = time.time() print "Generation completed in %s seconds" % (now - start)
def django_table_names(self, only_existing=False): """ Returns a list of all table names that have associated Django models and are in INSTALLED_APPS. If only_existing is True, the resulting list will only include the tables that actually exist in the database. """ from django.db import models, router tables = set() for app in models.get_apps(): for model in models.get_models(app): if not model._meta.managed: continue if not router.allow_migrate(self.connection.alias, model): continue tables.add(model._meta.db_table) tables.update(f.m2m_db_table() for f in model._meta.local_many_to_many) tables = list(tables) if only_existing: existing_tables = self.table_names() tables = [ t for t in tables if self.table_name_converter(t) in existing_tables ] return tables
def sequence_list(self): "Returns a list of information about all DB sequences for all models in all apps." from django.db import models, router apps = models.get_apps() sequence_list = [] for app in apps: for model in models.get_models(app): if not model._meta.managed: continue if model._meta.swapped: continue if not router.allow_migrate(self.connection.alias, model): continue for f in model._meta.local_fields: if isinstance(f, models.AutoField): sequence_list.append({'table': model._meta.db_table, 'column': f.column}) break # Only one AutoField is allowed per model, so don't bother continuing. for f in model._meta.local_many_to_many: # If this is an m2m using an intermediate table, # we don't need to reset the sequence. if f.rel.through is None: sequence_list.append({'table': f.m2m_db_table(), 'column': None}) return sequence_list
def get_models(app_labels): """ Gets a list of models for the given app labels, with some exceptions. TODO: If a required model is referenced, it should also be included. Or at least discovered with a get_or_create() call. """ from django.db.models import get_app, get_apps, get_model from django.db.models import get_models as get_all_models # These models are not to be output, e.g. because they can be generated automatically # TODO: This should be "appname.modelname" string EXCLUDED_MODELS = (ContentType, ) models = [] # If no app labels are given, return all if not app_labels: for app in get_apps(): models += [m for m in get_all_models(app) if m not in EXCLUDED_MODELS] # Get all relevant apps for app_label in app_labels: # If a specific model is mentioned, get only that model if "." in app_label: app_label, model_name = app_label.split(".", 1) models.append(get_model(app_label, model_name)) # Get all models for a given app else: models += [m for m in get_all_models(get_app(app_label)) if m not in EXCLUDED_MODELS] return models
def handle(self, *args, **options): try: cursor = connection.cursor() self.stdout.write('Building SQL bins into database..\n') for apps in get_apps(): sql_bin = os.path.join(os.path.dirname(apps.__file__), 'sql/bin') if os.path.isdir(sql_bin): self.stdout.write('SQL/BIN found in %s ...\n' % apps.__name__) for file in glob.glob("%s/*.sql" % sql_bin): self.stdout.write('Executing (%s) in %s\n' % (file, apps.__name__)) f = open(file, mode='r') sql = f.read() f.close() cursor.execute(sql) cursor.execute('commit') self.stdout.write('\n') self.stdout.write('Execution completed, exiting.\n') except Exception as e: raise CommandError('Exception: %s' % str(e))
def find_modules_for_script(script): """ find script module which contains 'run' attribute """ modules = [] # first look in apps for app in get_apps(): app_name = app.__name__.split(".")[:-1] # + ['fixtures'] for subdir in subdirs: mod = my_import(".".join(app_name + [subdir, script])) if mod: modules.append(mod) # try app.DIR.script import sa = script.split(".") for subdir in subdirs: nn = ".".join(sa[:-1] + [subdir, sa[-1]]) mod = my_import(nn) if mod: modules.append(mod) # try direct import if script.find(".") != -1: mod = my_import(script) if mod: modules.append(mod) return modules
def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]): """Test runner to support many DocTests *.txt files and TestUnits *.py using a setting TEST_FILES in app.tests module""" for app in get_apps(): test_mod = get_tests(app) if not test_mod or hasattr(test_mod, 'suite'): continue suites = [] # DocTest files for filename in getattr(test_mod, 'DOCTEST_FILES', []): try: suites.append(doctest.DocFileSuite( filename, package=test_mod, encoding='utf-8', )) except TypeError: suites.append(doctest.DocFileSuite( filename, package=test_mod, )) # Unit Tests modules for module in getattr(test_mod, 'UNITTEST_MODULES', []): suites.append(unittest.TestLoader().loadTestsFromModule(module)) # Sets the 'suites' attribute to test module if suites: print suites test_mod.suite = lambda: unittest.TestSuite(suites) return old_run_tests(test_labels, verbosity, interactive, extra_tests)
def handle(self, *labels, **options): if not labels or len(labels) > 1: raise CommandError('Enter one directory name.') label = labels[0] final_dest = osp.join(os.getcwd(), label) if osp.exists(final_dest): raise CommandError('Directory already exists') os.mkdir(final_dest) apps = get_apps() for x in apps: app_dir = osp.dirname(x.__file__) module = x.__name__ app = module.split('.')[-2] if app == 'admin': continue media_dir = osp.join(app_dir, "media", app) if not osp.isdir(media_dir): media_dir = osp.join(app_dir, "media") if osp.exists(media_dir): print "copy", media_dir, '->', osp.join(final_dest, app) shutil.copytree(media_dir, osp.join(final_dest, app))
def _get_fixture_dirs(self): """Returns the list of fixture directories. This is computed only once and cached. """ if not TestCase._fixture_dirs: app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage. for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) all_fixture_dirs = [ os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths ] TestCase._fixture_dirs = [ fixture_dir for fixture_dir in all_fixture_dirs if os.path.exists(fixture_dir) ] return TestCase._fixture_dirs
def handle(self, *args, **options): fields_data = [] if len(args) != 0: for arg in args: field_spec = arg.split('.') if len(field_spec) == 1: app = get_app(field_spec[0]) models = get_models(app) for model in models: fields_data += all_fsm_fields_data(model) elif len(field_spec) == 2: model = get_model(field_spec[0], field_spec[1]) fields_data += all_fsm_fields_data(model) elif len(field_spec) == 3: model = get_model(field_spec[0], field_spec[1]) fields_data.append((model._meta.get_field_by_name(field_spec[2])[0], model)) else: for app in get_apps(): for model in get_models(app): fields_data += all_fsm_fields_data(model) dotdata = generate_dot(fields_data) if options['outputfile']: self.render_output(dotdata, **options) else: print(dotdata)
def handle(self, *args, **options): # this is overridden so we can also do the trick on couchapps for app in get_apps(): couchdbkit_handler.copy_designs(app, temp='tmp', verbosity=2) for plugin in get_preindex_plugins(): plugin.copy_designs(temp='tmp')
def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]): """ Test runner which displays a code coverage report at the end of the run. """ django_test_runner = get_runner(settings) coverage.use_cache(0) coverage.start() results = django_test_runner(test_labels, verbosity, interactive, extra_tests) coverage.stop() coverage_modules = [] if test_labels: for label in test_labels: # Don't report coverage if you're only running a single # test case. if '.' not in label: app = get_app(label) coverage_modules.extend(get_all_coverage_modules(app)) else: for app in get_apps(): coverage_modules.extend(get_all_coverage_modules(app)) if coverage_modules: coverage.report(coverage_modules, show_missing=1) return results
def runjobs_by_signals(self, when, options): """ Run jobs from the signals """ # Thanks for Ian Holsman for the idea and code from django_extensions.management import signals from django.db import models from django.conf import settings verbosity = int(options.get('verbosity', 1)) for app_name in settings.INSTALLED_APPS: try: __import__(app_name + '.management', '', '', ['']) except ImportError: pass for app in models.get_apps(): if verbosity > 1: app_name = '.'.join(app.__name__.rsplit('.')[:-1]) print("Sending %s job signal for: %s" % (when, app_name)) if when == 'minutely': signals.run_minutely_jobs.send(sender=app, app=app) elif when == 'quarter_hourly': signals.run_quarter_hourly_jobs.send(sender=app, app=app) elif when == 'hourly': signals.run_hourly_jobs.send(sender=app, app=app) elif when == 'daily': signals.run_daily_jobs.send(sender=app, app=app) elif when == 'weekly': signals.run_weekly_jobs.send(sender=app, app=app) elif when == 'monthly': signals.run_monthly_jobs.send(sender=app, app=app) elif when == 'yearly': signals.run_yearly_jobs.send(sender=app, app=app)
def emit_post_migrate(verbosity, interactive, database): # Emit the post migrate signal. This allows individual applications to # respond as if the database had been migrated from scratch. all_models = [] for app in models.get_apps(): all_models.extend(router.get_migratable_models(app, database, include_auto_created=True)) emit_post_migrate_signal(set(all_models), verbosity, interactive, database)
def handle_noargs(self, **options): # retrieve sqlalchemy engine for selected database db = options.get('database') engine = get_engine(db) # retrieve schema schema = options.get('schema') if schema is not None: schema = Schema(schema) # account for setting overwrite if any metadatas = set([]) for modelmodul in get_apps(): for n, m in inspect.getmembers(modelmodul): if isinstance(m, MetaData): # if o is bound to another db than engine, ignore it if (m.bind is not None) and m.bind != engine: continue # if a schema was set # ignore metadata that don't belong to it if schema and m.schema != schema: continue metadatas.add(m) for metadata in metadatas: print("now synchronizing %s" % metadata) metadata.create_all(bind=engine)
def dump_data(request,appname): app_list = SortedDict() try: if request.POST: for appname in request.POST.getlist('apps'): app = get_app(appname) app_list[app] = None appname = 'choices' else: app = get_app(appname) app_list[app] = None except ImproperlyConfigured: if appname == 'all': for app in get_apps(): app_list[app] = None if(len(app_list) > 0): objects = [] for model in sort_dependencies(app_list.items()): if not model._meta.proxy and router.allow_syncdb(DEFAULT_DB_ALIAS, model): objects.extend(model._default_manager.using(DEFAULT_DB_ALIAS).all()) serializers.get_serializer('json') json = serializers.serialize('json', objects, indent=2,use_natural_keys=True) response = HttpResponse(json, mimetype='application/json'); response['Content-Disposition'] = 'attachment; filename=%s_%s_fixture.json' % (date.today().__str__(),appname) return response return render_to_response('diagnostic/dumpdata.html',context_instance=RequestContext(request))
def print_info(self): from django.db.models import get_apps app_list = get_apps() # output.append("App list: %s" % app_list) from django.core.management import sql from django.core.management.color import no_style def print_out(app, method_name): method = getattr(sql, method_name) try: output = method(app, no_style()) except: # sql_custom takes no stype... why? output = method(app) if output==[]: return print "--\n-- %s:\n--" % method_name for line in output: print line for app in app_list: print "--\n--", print "_"*77 print "-- %s\n--" % app.__name__ print_out(app, "sql_create") print_out(app, "sql_custom") print_out(app, "sql_indexes")
class Command(NoArgsCommand): option_list = syncdb.Command.option_list + ( make_option( '--migrate', action='store_true', dest='migrate', default=False, help= 'Tells South to also perform migrations after the sync. Default for during testing, and other internal calls.' ), make_option( '--all', action='store_true', dest='migrate_all', default=False, help= 'Makes syncdb work on all apps, even migrated ones. Be careful!'), ) if '--verbosity' not in [ opt.get_opt_string() for opt in syncdb.Command.option_list ]: option_list += (make_option( '--verbosity', action='store', dest='verbosity', default='1', type='choice', choices=['0', '1', '2'], help= 'Verbosity level; 0=minimal output, 1=normal output, 2=all output' ), ) help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created, except those which use migrations." def handle_noargs(self, migrate_all=False, **options): # Import the 'management' module within each installed app, to register # dispatcher events. # This is copied from Django, to fix bug #511. try: from django.utils.importlib import import_module except ImportError: pass # TODO: Remove, only for Django1.0 else: for app_name in settings.INSTALLED_APPS: try: import_module('.management', app_name) except ImportError, exc: msg = exc.args[0] if not msg.startswith( 'No module named') or 'management' not in msg: raise # Work out what uses migrations and so doesn't need syncing apps_needing_sync = [] apps_migrated = [] for app in models.get_apps(): app_label = get_app_label(app) if migrate_all: apps_needing_sync.append(app_label) else: try: migrations = migration.Migrations(app_label) except NoMigrations: # It needs syncing apps_needing_sync.append(app_label) else: # This is a migrated app, leave it apps_migrated.append(app_label) verbosity = int(options.get('verbosity', 0)) # Run syncdb on only the ones needed if verbosity: print "Syncing..." old_installed, settings.INSTALLED_APPS = settings.INSTALLED_APPS, apps_needing_sync old_app_store, cache.app_store = cache.app_store, SortedDict([ (k, v) for (k, v) in cache.app_store.items() if get_app_label(k) in apps_needing_sync ]) # This will allow the setting of the MySQL storage engine, for example. for db in dbs.values(): db.connection_init() # OK, run the actual syncdb syncdb.Command().execute(**options) settings.INSTALLED_APPS = old_installed cache.app_store = old_app_store # Migrate if needed if options.get('migrate', True): if verbosity: print "Migrating..." management.call_command('migrate', **options) # Be obvious about what we did if verbosity: print "\nSynced:\n > %s" % "\n > ".join(apps_needing_sync) if options.get('migrate', True): if verbosity: print "\nMigrated:\n - %s" % "\n - ".join(apps_migrated) else: if verbosity: print "\nNot synced (use migrations):\n - %s" % "\n - ".join( apps_migrated) print "(use ./manage.py migrate to migrate these)"
def handle(self, *fixture_labels, **options): from django.db.models import get_apps from django.core import serializers from django.db import connection, transaction from django.conf import settings self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # Keep a count of the installed objects and fixtures fixture_count = 0 object_count = 0 models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) app_fixtures = [ os.path.join(os.path.dirname(app.__file__), 'fixtures') for app in get_apps() ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) == 1: fixture_name = fixture_label formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if verbosity >= 2: if formats: print "Loading '%s' fixtures..." % fixture_name else: print "Skipping fixture '%s': %s is not a known serialization format" % ( fixture_name, format) if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: print "Checking %s for fixtures..." % humanize(fixture_dir) label_found = False for format in formats: serializer = serializers.get_serializer(format) if verbosity > 1: print "Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), format, fixture_name) try: full_path = os.path.join( fixture_dir, '.'.join([fixture_name, format])) fixture = open(full_path, 'r') if label_found: fixture.close() print self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) transaction.rollback() transaction.leave_transaction_management() return else: fixture_count += 1 if verbosity > 0: print "Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir)) try: objects = serializers.deserialize( format, fixture) for obj in objects: object_count += 1 models.add(obj.object.__class__) obj.save() label_found = True except Exception, e: fixture.close() transaction.rollback() transaction.leave_transaction_management() if show_traceback: import traceback traceback.print_exc() else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, str(e)))) return fixture.close() except: if verbosity >= 2: print "No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir)) if object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( self.style, models) if sequence_sql: if verbosity > 1: print "Resetting sequences" for line in sequence_sql: cursor.execute(line) transaction.commit() transaction.leave_transaction_management() if object_count == 0: if verbosity >= 2: print "No fixtures found." else: if verbosity > 0: print "Installed %d object(s) from %d fixture(s)" % ( object_count, fixture_count)
def handle(self, *app_labels, **options): from django.db.models import get_app, get_apps, get_model output_folder = options.get('output_folder') print "Output folder:", output_folder print "NOTE: See --output-folder option" max_records_per_chunk = options.get('max_records_per_chunk') format = options.get('format') indent = options.get('indent') using = options.get('database') excludes = options.get('exclude') show_traceback = options.get('traceback') use_natural_keys = options.get('use_natural_keys') use_base_manager = options.get('use_base_manager') excluded_apps = set() excluded_models = set() for exclude in excludes: if '.' in exclude: app_label, model_name = exclude.split('.', 1) model_obj = get_model(app_label, model_name) if not model_obj: raise CommandError('Unknown model in excludes: %s' % exclude) excluded_models.add(model_obj) else: try: app_obj = get_app(exclude) excluded_apps.add(app_obj) except ImproperlyConfigured: raise CommandError('Unknown app in excludes: %s' % exclude) if len(app_labels) == 0: app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps) else: app_list = SortedDict() for label in app_labels: try: app_label, model_label = label.split('.') try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue model = get_model(app_label, model_label) if model is None: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) if app in app_list.keys(): if app_list[app] and model not in app_list[app]: app_list[app].append(model) else: app_list[app] = [model] except ValueError: # This is just an app - no model qualifier app_label = label try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue app_list[app] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) # Now collate the objects to be serialized. objects = [] model_count = 1000 chunk_count = 1000 for model in sort_dependencies(app_list.items()): model_count += 1 if model in excluded_models: continue if not model._meta.proxy and router.allow_migrate(using, model): if use_base_manager: objects.extend(model._base_manager.using(using).all()) else: items_total = model._default_manager.using(using).count() chunks_total = (items_total / max_records_per_chunk) +1 for chunk_num in range(0, chunks_total): output_objects = model._default_manager.using(using).all().order_by('id')[chunk_num*max_records_per_chunk:(chunk_num+1)*max_records_per_chunk] if output_objects: chunk_count += 1 dump_file_name = output_folder + "/%d_%d.json" % (model_count, chunk_count) print "Dumping file: %s [%d]" % (dump_file_name, chunks_total) output = serializers.serialize(format, output_objects, indent=indent, use_natural_keys=use_natural_keys) with open(dump_file_name, "w") as dumpfile: dumpfile.write(output) return ''
def handle(self, *app_labels, **options): from django.db.models import get_app, get_apps, get_model format = options.get('format') indent = options.get('indent') using = options.get('database') excludes = options.get('exclude') show_traceback = options.get('traceback') use_natural_keys = options.get('use_natural_keys') use_base_manager = options.get('use_base_manager') pks = options.get('primary_keys') if pks: primary_keys = pks.split(',') else: primary_keys = [] excluded_apps = set() excluded_models = set() for exclude in excludes: if '.' in exclude: app_label, model_name = exclude.split('.', 1) model_obj = get_model(app_label, model_name) if not model_obj: raise CommandError('Unknown model in excludes: %s' % exclude) excluded_models.add(model_obj) else: try: app_obj = get_app(exclude) excluded_apps.add(app_obj) except ImproperlyConfigured: raise CommandError('Unknown app in excludes: %s' % exclude) if len(app_labels) == 0: if primary_keys: raise CommandError( "You can only use --pks option with one model") app_list = OrderedDict( (app, None) for app in get_apps() if app not in excluded_apps) else: if len(app_labels) > 1 and primary_keys: raise CommandError( "You can only use --pks option with one model") app_list = OrderedDict() for label in app_labels: try: app_label, model_label = label.split('.') try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue model = get_model(app_label, model_label) if model is None: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) if app in app_list.keys(): if app_list[app] and model not in app_list[app]: app_list[app].append(model) else: app_list[app] = [model] except ValueError: if primary_keys: raise CommandError( "You can only use --pks option with one model") # This is just an app - no model qualifier app_label = label try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue app_list[app] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) def get_objects(): # Collate the objects to be serialized. for model in sort_dependencies(app_list.items()): if model in excluded_models: continue if not model._meta.proxy and router.allow_migrate( using, model): if use_base_manager: objects = model._base_manager else: objects = model._default_manager queryset = objects.using(using).order_by( model._meta.pk.name) if primary_keys: queryset = queryset.filter(pk__in=primary_keys) for obj in queryset.iterator(): yield obj try: self.stdout.ending = None serializers.serialize(format, get_objects(), indent=indent, use_natural_keys=use_natural_keys, stream=self.stdout) except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def handle_noargs(self, migrate_all=False, **options): # Import the 'management' module within each installed app, to register # dispatcher events. # This is copied from Django, to fix bug #511. try: from django.utils.importlib import import_module except ImportError: pass # TODO: Remove, only for Django1.0 else: for app_name in settings.INSTALLED_APPS: try: import_module('.management', app_name) except ImportError as exc: msg = exc.args[0] if not msg.startswith('No module named') or 'management' not in msg: raise # Work out what uses migrations and so doesn't need syncing apps_needing_sync = [] apps_migrated = [] for app in models.get_apps(): app_label = get_app_label(app) if migrate_all: apps_needing_sync.append(app_label) else: try: migrations = migration.Migrations(app_label) except NoMigrations: # It needs syncing apps_needing_sync.append(app_label) else: # This is a migrated app, leave it apps_migrated.append(app_label) verbosity = int(options.get('verbosity', 0)) # Run syncdb on only the ones needed if verbosity: print("Syncing...") old_installed, settings.INSTALLED_APPS = settings.INSTALLED_APPS, apps_needing_sync old_app_store, cache.app_store = cache.app_store, SortedDict([ (k, v) for (k, v) in cache.app_store.items() if get_app_label(k) in apps_needing_sync ]) # This will allow the setting of the MySQL storage engine, for example. for db in dbs.values(): db.connection_init() # OK, run the actual syncdb syncdb.Command().execute(**options) settings.INSTALLED_APPS = old_installed cache.app_store = old_app_store # Migrate if needed if options.get('migrate', True): if verbosity: print("Migrating...") # convert from store_true to store_false options['no_initial_data'] = not options.get('load_initial_data', True) management.call_command('migrate', **options) # Be obvious about what we did if verbosity: print("\nSynced:\n > %s" % "\n > ".join(apps_needing_sync)) if options.get('migrate', True): if verbosity: print("\nMigrated:\n - %s" % "\n - ".join(apps_migrated)) else: if verbosity: print("\nNot synced (use migrations):\n - %s" % "\n - ".join(apps_migrated)) print("(use ./manage.py migrate to migrate these)")
class Command(NoArgsCommand): """ It validates all models that are G11nBase subclass. """ help = "Validates all models that are G11nBase subclass." def handle(self, *args, **options): for app_name in settings.INSTALLED_APPS: #controlla tutte le app in INSTALLED_APP try: import_module('.management', app_name) # Prova ad importare l'applicazione except ImportError, exc: msg = exc.args[0] if not msg.startswith( 'No module named' ) or 'management' not in msg: #se non ci riesce lancia eccezione raise for app in models.get_apps(): # cicla tutte le applicazioni for m in models.get_models( app): # per ogni modello nell'applicazione corrente if issubclass( m, G11nModel ): # se e' subclass di G11nModel deve controllare che estende una classe astratta e non conreta for base in m.__bases__: if issubclass(base, G11nModel) and not base._meta.abstract: raise CommandError( "Error in %s.%s . G11nModel can extend only abstract Model" % (m.__module__, m.__name__)) elif issubclass( m, G11nBase): # se il modello e' sottoclasse di G11nBase if not hasattr( m.G11nMeta, "g11n" ): #controlla che abbia il parametro g11n nella meta class G11nMeta raise CommandError( "Error in %s.%s . G11nMeta must have field with the name g11n" % (m.__module__, m.__name__)) elif not hasattr( m.G11nMeta, "fieldname" ): #controlla che abbia il parametro fieldname nella meta class G11nMeta raise CommandError( "Error in %s.%s . G11nMeta must have field with the name fieldname" % (m.__module__, m.__name__)) elif not m.G11nMeta.g11n: #controlla che il parametro g11n non sia nullo nella meta class G11nMeta raise CommandError( "Error in %s. %s is a G11nBase model and it needs a meta class G11nMeta." % (m.__module__, m.__name__)) elif not hasattr( app, m.G11nMeta.g11n ): #controlla che esista il modello definito nel parametro g11n della meta class G11nMeta raise CommandError( "Error in %s Model with the name %s doesn't exists." % (m.__module__, m.G11nMeta.g11n)) else: g11nmodel = getattr( app, m.G11nMeta.g11n ) #importa il modello ereditato da G11nModel if not issubclass( g11nmodel, G11nModel ): #controlla che effettivamente sia un G11nModel raise CommandError( "Error in %s . %s is not a G11nModel subclass." % (m.__module__, g11nmodel.__name__)) else: #se G11nBase passa la validazione e siamo in un G11nModel allora controlla le sue configurazioni foreign_keys = [] properties = [] for k, v in m.__dict__.items(): if type(v) == property: properties.append(k) for field in g11nmodel._meta.fields: #per ogni field del g11nmodel if field.get_internal_type( ) == "ForeignKey" and m.__name__ == field.rel.to.__name__: # se il field e' una FK il nome del modello G11nBase == al nome del modello al quale g11nmodel si riferisce per questo field foreign_keys.append( field ) # aggiunge alla lista questo campo if m.G11nMeta.fieldname != field.name: #se il nome del field definito nella meta class G11nMeta != dal nome di questo campo allora lancia eccezione raise CommandError( "Error in %s.%s . G11nMeta.fieldname must be like a field's name in %s." % (m.__module__, m.__name__, g11nmodel.__name__)) if field.name in properties: raise CommandError( "Error in %s . %s has the property '%s' with the same name of a %s's attribute." % (m.__module__, m.__name__, field.name, g11nmodel.__name__)) if not foreign_keys: # se la lista e' vuota, significa che il g11nmodel non ha una FK verso il G11nBase raise CommandError( "Error in %s . %s has not a field related to %s." % (m.__module__, g11nmodel.__name__, m.__name__)) self.stdout.write("0 errors found in all G11nBase models\n")
def handle(self, *args, **options): """Handle the command.""" if len(args) != 1: raise CommandError("You must specify a filename on the command " "line.") filename = args[0] if not os.path.exists(filename): raise CommandError("%s does not exist." % filename) try: import django_reset except ImportError: raise CommandError("Before using this command, you need to " "install the 'django-reset' package") confirm = input(""" This will wipe out your existing database prior to loading. It is highly recommended that you have a full SQL database dump in case things go wrong. You should only use this if you're migrating from one type of database to another, with the same version of Review Board on each. Are you sure you want to continue?" Type 'yes' to continue, or 'no' to cancel: """) if confirm != 'yes': return apps = [app.__name__.split('.')[-2] for app in get_apps()] os.system('./reviewboard/manage.py reset --noinput %s' % ' '.join(apps)) transaction_setup = False try: with open(filename, 'r') as f: line = f.readline() m = re.match("^# dbdump v(\d+) - (\d+) objects$", line) if not m: raise CommandError("Unknown dump format\n") version = int(m.group(1)) totalobjs = int(m.group(2)) i = 0 prev_pct = -1 if version != 1: raise CommandError("Unknown dump version\n") transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) transaction_setup = True self.stdout.write("Importing new style dump format (v%s)" % version) for line in f: if line[0] == "{": for obj in serializers.deserialize( "json", "[%s]" % line): try: obj.save() except Exception as e: self.stderr.write("Error: %s\n" % e) self.stderr.write("Line %s: '%s'" % (i, line)) elif line[0] != "#": self.stderr.write("Junk data on line %s" % i) db.reset_queries() i += 1 pct = (i * 100 / totalobjs) if pct != prev_pct: self.stdout.write(" [%s%%]\r" % pct) self.stdout.flush() prev_pct = pct transaction.commit() transaction.leave_transaction_management() except Exception as e: raise CommandError("Problem installing '%s': %s\n" % (filename, e)) if transaction_setup: transaction.rollback() transaction.leave_transaction_management() self.stdout.write('\nDone.')
def validate(cls, model): """ Does basic ModelAdmin option validation. Calls custom validation classmethod in the end if it is provided in cls. The signature of the custom validation classmethod should be: def validate(cls, model). """ # Before we can introspect models, they need to be fully loaded so that # inter-relations are set up correctly. We force that here. models.get_apps() opts = model._meta validate_base(cls, model) # list_display if hasattr(cls, 'list_display'): check_isseq(cls, 'list_display', cls.list_display) for idx, field in enumerate(cls.list_display): if not callable(field): if not hasattr(cls, field): if not hasattr(model, field): try: opts.get_field(field) except models.FieldDoesNotExist: raise ImproperlyConfigured( "%s.list_display[%d], %r is not a callable or an attribute of %r or found in the model %r." % (cls.__name__, idx, field, cls.__name__, model._meta.object_name)) else: # getattr(model, field) could be an X_RelatedObjectsDescriptor f = fetch_attr(cls, model, opts, "list_display[%d]" % idx, field) if isinstance(f, models.ManyToManyField): raise ImproperlyConfigured( "'%s.list_display[%d]', '%s' is a ManyToManyField which is not supported." % (cls.__name__, idx, field)) # list_display_links if hasattr(cls, 'list_display_links'): check_isseq(cls, 'list_display_links', cls.list_display_links) for idx, field in enumerate(cls.list_display_links): fetch_attr(cls, model, opts, 'list_display_links[%d]' % idx, field) if field not in cls.list_display: raise ImproperlyConfigured( "'%s.list_display_links[%d]'" "refers to '%s' which is not defined in 'list_display'." % (cls.__name__, idx, field)) # list_filter if hasattr(cls, 'list_filter'): check_isseq(cls, 'list_filter', cls.list_filter) for idx, field in enumerate(cls.list_filter): get_field(cls, model, opts, 'list_filter[%d]' % idx, field) # list_per_page = 100 if hasattr(cls, 'list_per_page') and not isinstance(cls.list_per_page, int): raise ImproperlyConfigured("'%s.list_per_page' should be a integer." % cls.__name__) # list_editable if hasattr(cls, 'list_editable') and cls.list_editable: check_isseq(cls, 'list_editable', cls.list_editable) for idx, field_name in enumerate(cls.list_editable): try: field = opts.get_field_by_name(field_name)[0] except models.FieldDoesNotExist: raise ImproperlyConfigured( "'%s.list_editable[%d]' refers to a " "field, '%s', not defined on %s." % (cls.__name__, idx, field_name, model.__name__)) if field_name not in cls.list_display: raise ImproperlyConfigured( "'%s.list_editable[%d]' refers to " "'%s' which is not defined in 'list_display'." % (cls.__name__, idx, field_name)) if field_name in cls.list_display_links: raise ImproperlyConfigured( "'%s' cannot be in both '%s.list_editable'" " and '%s.list_display_links'" % (field_name, cls.__name__, cls.__name__)) if not cls.list_display_links and cls.list_display[ 0] in cls.list_editable: raise ImproperlyConfigured( "'%s.list_editable[%d]' refers to" " the first field in list_display, '%s', which can't be" " used unless list_display_links is set." % (cls.__name__, idx, cls.list_display[0])) if not field.editable: raise ImproperlyConfigured( "'%s.list_editable[%d]' refers to a " "field, '%s', which isn't editable through the admin." % (cls.__name__, idx, field_name)) # search_fields = () if hasattr(cls, 'search_fields'): check_isseq(cls, 'search_fields', cls.search_fields) # date_hierarchy = None if cls.date_hierarchy: f = get_field(cls, model, opts, 'date_hierarchy', cls.date_hierarchy) if not isinstance(f, (models.DateField, models.DateTimeField)): raise ImproperlyConfigured( "'%s.date_hierarchy is " "neither an instance of DateField nor DateTimeField." % cls.__name__) # ordering = None if cls.ordering: check_isseq(cls, 'ordering', cls.ordering) for idx, field in enumerate(cls.ordering): if field == '?' and len(cls.ordering) != 1: raise ImproperlyConfigured( "'%s.ordering' has the random " "ordering marker '?', but contains other fields as " "well. Please either remove '?' or the other fields." % cls.__name__) if field == '?': continue if field.startswith('-'): field = field[1:] # Skip ordering in the format field1__field2 (FIXME: checking # this format would be nice, but it's a little fiddly). if '__' in field: continue get_field(cls, model, opts, 'ordering[%d]' % idx, field) # list_select_related = False # save_as = False # save_on_top = False for attr in ('list_select_related', 'save_as', 'save_on_top'): if not isinstance(getattr(cls, attr), bool): raise ImproperlyConfigured("'%s.%s' should be a boolean." % (cls.__name__, attr)) # inlines = [] if hasattr(cls, 'inlines'): check_isseq(cls, 'inlines', cls.inlines) for idx, inline in enumerate(cls.inlines): if not issubclass(inline, BaseModelAdmin): raise ImproperlyConfigured("'%s.inlines[%d]' does not inherit " "from BaseModelAdmin." % (cls.__name__, idx)) if not inline.model: raise ImproperlyConfigured("'model' is a required attribute " "of '%s.inlines[%d]'." % (cls.__name__, idx)) if not issubclass(inline.model, models.Model): raise ImproperlyConfigured("'%s.inlines[%d].model' does not " "inherit from models.Model." % (cls.__name__, idx)) validate_base(inline, inline.model) validate_inline(inline, cls, model)
def handle(self, *fixture_labels, **options): from django.db.models import get_apps from django.core import serializers from django.db import connection, transaction from django.conf import settings self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 object_count = 0 models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) self.disable_forward_ref_checks() class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: file, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_fixtures = [os.path.join(os.path.dirname(app.__file__), 'fixtures') for app in get_apps()] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: print("Loading '%s' fixtures..." % fixture_name) else: self.enable_forward_ref_checks(cursor) sys.stderr.write( self.style.ERROR("Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format))) transaction.rollback() transaction.leave_transaction_management() return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: print("Checking %s for fixtures..." % humanize(fixture_dir)) label_found = False for format in formats: for compression_format in compression_formats: if compression_format: file_name = '.'.join([fixture_name, format, compression_format]) else: file_name = '.'.join([fixture_name, format]) if verbosity > 1: print("Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') if label_found: fixture.close() self.enable_forward_ref_checks(cursor) print(self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir)))) transaction.rollback() transaction.leave_transaction_management() return else: fixture_count += 1 objects_in_fixture = 0 if verbosity > 0: print("Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir))) try: objects = serializers.deserialize(format, fixture) for obj in objects: objects_in_fixture += 1 self.handle_ref_checks(cursor, obj) models.add(obj.object.__class__) obj.save() object_count += objects_in_fixture label_found = True except (SystemExit, KeyboardInterrupt): self.enable_forward_ref_checks(cursor) raise except Exception: import traceback fixture.close() self.enable_forward_ref_checks(cursor) transaction.rollback() transaction.leave_transaction_management() if show_traceback: traceback.print_exc() else: sys.stderr.write( self.style.ERROR("Problem installing fixture '%s': %s\n" % (full_path, ''.join(traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))))) return fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: self.enable_forward_ref_checks(cursor) sys.stderr.write( self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)" % (fixture_name))) transaction.rollback() transaction.leave_transaction_management() return except Exception as e: if verbosity > 1: print("No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir))) self.enable_forward_ref_checks(cursor) # If we found even one object in a fixture, we need to reset the # database sequences. if object_count > 0: sequence_sql = connection.ops.sequence_reset_sql(self.style, models) if sequence_sql: if verbosity > 1: print("Resetting sequences") for line in sequence_sql: cursor.execute(line) if commit: transaction.commit() transaction.leave_transaction_management() if object_count == 0: if verbosity > 1: print("No fixtures found.") else: if verbosity > 0: print("Installed %d object(s) from %d fixture(s)" % (object_count, fixture_count)) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. if commit: connection.close()
def handle(self, *app_labels, **options): from django.db.models import get_app, get_apps, get_model format = options.get('format') indent = options.get('indent') using = options.get('database') excludes = options.get('exclude') show_traceback = options.get('traceback') use_natural_keys = options.get('use_natural_keys') use_base_manager = options.get('use_base_manager') excluded_apps = set() excluded_models = set() for exclude in excludes: if '.' in exclude: app_label, model_name = exclude.split('.', 1) model_obj = get_model(app_label, model_name) if not model_obj: raise CommandError('Unknown model in excludes: %s' % exclude) excluded_models.add(model_obj) else: try: app_obj = get_app(exclude) excluded_apps.add(app_obj) except ImproperlyConfigured: raise CommandError('Unknown app in excludes: %s' % exclude) if len(app_labels) == 0: app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps) else: app_list = SortedDict() for label in app_labels: try: app_label, model_label = label.split('.') try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue model = get_model(app_label, model_label) if model is None: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) if app in app_list.keys(): if app_list[app] and model not in app_list[app]: app_list[app].append(model) else: app_list[app] = [model] except ValueError: # This is just an app - no model qualifier app_label = label try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue app_list[app] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) # Now collate the objects to be serialized. objects = [] for model in sort_dependencies(app_list.items()): if model in excluded_models: continue if not model._meta.proxy and router.allow_syncdb(using, model): if use_base_manager: objects.extend(model._base_manager.using(using).all()) else: objects.extend(model._default_manager.using(using).all()) try: return serializers.serialize(format, objects, indent=indent, use_natural_keys=use_natural_keys) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
class Command(NoArgsCommand): option_list = NoArgsCommand.option_list + ( make_option( '--noinput', action='store_false', dest='interactive', default=True, help='Tells Django to NOT prompt the user for input of any kind.'), make_option('--database', action='store', dest='database', default=DEFAULT_DB_ALIAS, help='Nominates a database to synchronize. ' 'Defaults to the "default" database.'), ) help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created." def handle_noargs(self, **options): verbosity = int(options.get('verbosity', 1)) interactive = options.get('interactive') show_traceback = options.get('traceback', False) # Stealth option -- 'load_initial_data' is used by the testing setup # process to disable initial fixture loading. load_initial_data = options.get('load_initial_data', True) self.style = no_style() # Import the 'management' module within each installed app, to register # dispatcher events. for app_name in settings.INSTALLED_APPS: try: import_module('.management', app_name) except ImportError, exc: # This is slightly hackish. We want to ignore ImportErrors # if the "management" module itself is missing -- but we don't # want to ignore the exception if the management module exists # but raises an ImportError for some reason. The only way we # can do this is to check the text of the exception. Note that # we're a bit broad in how we check the text, because different # Python implementations may not use the same text. # CPython uses the text "No module named management" # PyPy uses "No module named myproject.myapp.management" msg = exc.args[0] if not msg.startswith( 'No module named') or 'management' not in msg: raise db = options.get('database', DEFAULT_DB_ALIAS) connection = connections[db] cursor = connection.cursor() # Get a list of already installed *models* so that references work right. tables = connection.introspection.table_names() seen_models = connection.introspection.installed_models(tables) created_models = set() pending_references = {} # Build the manifest of apps and models that are to be synchronized all_models = [(app.__name__.split('.')[-2], [ m for m in models.get_models(app, include_auto_created=True) if router.allow_syncdb(db, m) ]) for app in models.get_apps()] def model_installed(model): opts = model._meta converter = connection.introspection.table_name_converter return not ((converter(opts.db_table) in tables) or (opts.auto_created and converter( opts.auto_created._meta.db_table) in tables)) manifest = SortedDict((app_name, filter(model_installed, model_list)) for app_name, model_list in all_models) # Create the tables for each model for app_name, model_list in manifest.items(): for model in model_list: # Create the model's database table, if it doesn't already exist. if verbosity >= 2: print "Processing %s.%s model" % (app_name, model._meta.object_name) sql, references = connection.creation.sql_create_model( model, self.style, seen_models) seen_models.add(model) created_models.add(model) for refto, refs in references.items(): pending_references.setdefault(refto, []).extend(refs) if refto in seen_models: sql.extend( connection.creation.sql_for_pending_references( refto, self.style, pending_references)) sql.extend( connection.creation.sql_for_pending_references( model, self.style, pending_references)) if verbosity >= 1 and sql: print "Creating table %s" % model._meta.db_table for statement in sql: cursor.execute(statement) tables.append( connection.introspection.table_name_converter( model._meta.db_table)) transaction.commit_unless_managed(using=db) # Send the post_syncdb signal, so individual apps can do whatever they need # to do at this point. emit_post_sync_signal(created_models, verbosity, interactive, db) # The connection may have been closed by a syncdb handler. cursor = connection.cursor() # Install custom SQL for the app (but only if this # is a model we've just created) for app_name, model_list in manifest.items(): for model in model_list: if model in created_models: custom_sql = custom_sql_for_model(model, self.style, connection) if custom_sql: if verbosity >= 1: print "Installing custom SQL for %s.%s model" % ( app_name, model._meta.object_name) try: for sql in custom_sql: cursor.execute(sql) except Exception, e: sys.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \ (app_name, model._meta.object_name, e)) if show_traceback: import traceback traceback.print_exc() transaction.rollback_unless_managed(using=db) else: transaction.commit_unless_managed(using=db) else: if verbosity >= 2: print "No custom SQL for %s.%s model" % ( app_name, model._meta.object_name)
def generate_dot(app_labels, **kwargs): disable_fields = kwargs.get('disable_fields', False) include_models = parse_file_or_list(kwargs.get('include_models', "")) all_applications = kwargs.get('all_applications', False) use_subgraph = kwargs.get('group_models', False) verbose_names = kwargs.get('verbose_names', False) inheritance = kwargs.get('inheritance', False) language = kwargs.get('language', None) if language is not None: activate_language(language) exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', "")) exclude_models = parse_file_or_list(kwargs.get('exclude_models', "")) def skip_field(field): if exclude_columns: if verbose_names and field.verbose_name: if field.verbose_name in exclude_columns: return True if field.name in exclude_columns: return True return False t = loader.get_template('django_extensions/graph_models/head.html') c = Context({}) dot = t.render(c) apps = [] if all_applications: apps = models.get_apps() for app_label in app_labels: app = models.get_app(app_label) if not app in apps: apps.append(app) graphs = [] for app in apps: graph = Context({ 'name': '"%s"' % app.__name__, 'app_name': "%s" % '.'.join(app.__name__.split('.')[:-1]), 'cluster_app_name': "cluster_%s" % app.__name__.replace(".", "_"), 'disable_fields': disable_fields, 'use_subgraph': use_subgraph, 'models': [] }) appmodels = get_models(app) abstract_models = [] for appmodel in appmodels: abstract_models = abstract_models + [ abstract_model for abstract_model in appmodel.__bases__ if hasattr( abstract_model, '_meta') and abstract_model._meta.abstract ] abstract_models = list(set(abstract_models)) # remove duplicates appmodels = abstract_models + appmodels for appmodel in appmodels: appmodel_abstracts = [ abstract_model.__name__ for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract ] # collect all attribs of abstract superclasses def getBasesAbstractFields(c): _abstract_fields = [] for e in c.__bases__: if hasattr(e, '_meta') and e._meta.abstract: _abstract_fields.extend(e._meta.fields) _abstract_fields.extend(getBasesAbstractFields(e)) return _abstract_fields abstract_fields = getBasesAbstractFields(appmodel) model = { 'app_name': appmodel.__module__.replace(".", "_"), 'name': appmodel.__name__, 'abstracts': appmodel_abstracts, 'fields': [], 'relations': [] } # consider given model name ? def consider(model_name): if exclude_models and model_name in exclude_models: return False return not include_models or model_name in include_models if not consider(appmodel._meta.object_name): continue if verbose_names and appmodel._meta.verbose_name: model['label'] = appmodel._meta.verbose_name else: model['label'] = model['name'] # model attributes def add_attributes(field): if verbose_names and field.verbose_name: label = field.verbose_name else: label = field.name t = type(field).__name__ if isinstance(field, (OneToOneField, ForeignKey)): t += " ({0})".format(field.rel.field_name) # TODO: ManyToManyField, GenericRelation model['fields'].append({ 'name': field.name, 'label': label, 'type': t, 'blank': field.blank, 'abstract': field in abstract_fields, }) # Find all the real attributes. Relations are depicted as graph edges instead of attributes attributes = [ field for field in appmodel._meta.local_fields if not isinstance(field, RelatedField) ] # find primary key and print it first, ignoring implicit id if other pk exists pk = appmodel._meta.pk if not appmodel._meta.abstract and pk in attributes: add_attributes(pk) for field in attributes: if skip_field(field): continue if not field.primary_key: add_attributes(field) # FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph? #if appmodel._meta.many_to_many: # for field in appmodel._meta.many_to_many: # if skip_field(field): # continue # add_attributes(field) # relations def add_relation(field, extras=""): if verbose_names and field.verbose_name: label = field.verbose_name else: label = field.name # show related field name if hasattr(field, 'related_query_name'): label += ' (%s)' % field.related_query_name() # handle self-relationships if field.rel.to == 'self': target_model = field.model else: target_model = field.rel.to _rel = { 'target_app': target_model.__module__.replace('.', '_'), 'target': target_model.__name__, 'type': type(field).__name__, 'name': field.name, 'label': label, 'arrows': extras, 'needs_node': True } if _rel not in model['relations'] and consider(_rel['target']): model['relations'].append(_rel) for field in appmodel._meta.local_fields: if field.attname.endswith( '_ptr_id' ): # excluding field redundant with inheritance relation continue if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields continue if skip_field(field): continue if isinstance(field, OneToOneField): add_relation(field, '[arrowhead=none, arrowtail=none]') elif isinstance(field, ForeignKey): add_relation(field, '[arrowhead=none, arrowtail=dot]') for field in appmodel._meta.local_many_to_many: if skip_field(field): continue if isinstance(field, ManyToManyField): if (getattr(field, 'creates_table', False) or # django 1.1. (hasattr(field.rel.through, '_meta') and field.rel.through._meta.auto_created)): # django 1.2 add_relation( field, '[arrowhead=dot arrowtail=dot, dir=both]') elif isinstance(field, GenericRelation): add_relation( field, mark_safe( '[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]' )) if inheritance: # add inheritance arrows for parent in appmodel.__bases__: if hasattr(parent, "_meta"): # parent is a model l = "multi-table" if parent._meta.abstract: l = "abstract" if appmodel._meta.proxy: l = "proxy" l += r"\ninheritance" _rel = { 'target_app': parent.__module__.replace(".", "_"), 'target': parent.__name__, 'type': "inheritance", 'name': "inheritance", 'label': l, 'arrows': '[arrowhead=empty, arrowtail=none]', 'needs_node': True } # TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes. if _rel not in model['relations'] and consider( _rel['target']): model['relations'].append(_rel) graph['models'].append(model) graphs.append(graph) nodes = [] for graph in graphs: nodes.extend([e['name'] for e in graph['models']]) for graph in graphs: # don't draw duplication nodes because of relations for model in graph['models']: for relation in model['relations']: if relation['target'] in nodes: relation['needs_node'] = False # render templates t = loader.get_template('django_extensions/graph_models/body.html') dot += '\n' + t.render(graph) for graph in graphs: t = loader.get_template('django_extensions/graph_models/rel.html') dot += '\n' + t.render(graph) t = loader.get_template('django_extensions/graph_models/tail.html') c = Context({}) dot += '\n' + t.render(c) return dot
def handle(self, *fixture_labels, **options): ''' All database code is unnecessary but remove it will broke. ''' #TODO remove database, connection code and stuff using = options.get('database') connection = connections[using] self.style = no_style() if not len(fixture_labels): print("No database fixture specified. Please provide the path of at least one fixture in the command line.\n") return verbosity = int(options.get('verbosity')) show_traceback = options.get('traceback') # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 models = set() humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths] instances = [] apps = set() try: #with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity >= 2: print("Loading '%s' fixtures...\n" % fixture_name) else: print("Problem installing fixture '%s': %s is not a known serialization format.\n" % (fixture_name, format)) if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity >= 2: self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join( p for p in [ fixture_name, database, format, compression_format ] if p ) if verbosity >= 3: self.stdout.write("Trying %s for %s fixture '%s'...\n" % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') except IOError: if verbosity >= 2: self.stdout.write("No %s fixture '%s' in %s.\n" % \ (format, fixture_name, humanize(fixture_dir))) else: try: if label_found: print("Multiple fixtures named '%s' in %s. Aborting.\n" % (fixture_name, humanize(fixture_dir))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) return fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if verbosity >= 2: self.stdout.write("Installing %s fixture '%s' from %s.\n" % \ (format, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize(format, fixture, using=using) for obj in objects: objects_in_fixture += 1 if router.allow_syncdb(using, obj.object.__class__): loaded_objects_in_fixture += 1 models.add(obj.object.__class__) try: obj.object.save = partial(memory_save, obj.object) #if obj.object.__class__.__name__ == "PedidoExame": # import ipdb; ipdb.set_trace() ### XXX BREAKPOINT obj.object.save() instances.append(obj) except (DatabaseError, IntegrityError), e: msg = "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': e } raise e.__class__, e.__class__(msg), sys.exc_info()[2] loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True finally: fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: print("No fixture data found for '%s'. (File format may be invalid.)\n" % (fixture_name)) if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) return # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in models]
def handle(self, *fixture_labels, **options): """ Main method of a Django command """ from django.db.models import get_apps from django.core import serializers from django.conf import settings self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # Keep a count of the installed objects and fixtures fixture_count = 0 object_count = 0 objects_per_fixture = [] models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() app_fixtures = [ os.path.join(os.path.dirname(app.__file__), 'fixtures') for app in get_apps() ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) == 1: fixture_name = fixture_label formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: print("Loading '%s' fixtures..." % fixture_name) else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format))) transaction.rollback() return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: print("Checking %s for fixtures..." % humanize(fixture_dir)) label_found = False for format in formats: if verbosity > 1: print("Trying %s for %s fixture '%s'..." % (humanize(fixture_dir), format, fixture_name)) try: full_path = os.path.join( fixture_dir, '.'.join([fixture_name, format])) fixture = open(full_path, 'r') if label_found: fixture.close() print( self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir)))) transaction.rollback() return else: fixture_count += 1 objects_per_fixture.append(0) if verbosity > 0: print("Installing %s fixture '%s' from %s." % (format, fixture_name, humanize(fixture_dir))) try: objects_to_keep = {} objects = serializers.deserialize( format, fixture) for obj in objects: object_count += 1 objects_per_fixture[-1] += 1 class_ = obj.object.__class__ if class_ not in objects_to_keep: objects_to_keep[class_] = set() objects_to_keep[class_].add(obj.object) models.add(class_) obj.save() if options.get('remove'): self.remove_objects_not_in( objects_to_keep, verbosity) label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() transaction.rollback() if show_traceback: traceback.print_exc() else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, traceback.format_exc()))) return fixture.close() except: if verbosity > 1: print( "No %s fixture '%s' in %s." % (format, fixture_name, humanize(fixture_dir))) # If any of the fixtures we loaded contain 0 objects, assume that an # error was encountered during fixture loading. if 0 in objects_per_fixture: sys.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)" % fixture_name)) transaction.rollback() return # If we found even one object in a fixture, we need to reset the # database sequences. if object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( self.style, models) if sequence_sql: if verbosity > 1: print("Resetting sequences") for line in sequence_sql: cursor.execute(line) transaction.commit() if object_count == 0: if verbosity > 1: print("No fixtures found.") else: if verbosity > 0: print("Installed %d object(s) from %d fixture(s)" % (object_count, fixture_count)) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. connection.close()
def handle(self, *args, **options): for app in get_apps(): couchdbkit_handler.sync(app, verbosity=2)
def handle(self, *args, **options): try: migrations_dir = settings.DMIGRATIONS_DIR except AttributeError: print "You need to add DMIGRATIONS_DIR to your settings" return migration_db = MigrationDb(directory = migrations_dir) migration_state = MigrationState( migration_db = migration_db, dev = options.get('dev') ) verbosity = int(options.get('verbosity', 1)) if not args or args[0] == 'help': self.print_help(sys.argv[0], 'dmigrate') return elif args[0] in 'all all_hard up down upto downto to apply unapply'.split(): migration_state.init() for (migration_name, action) in migration_state.plan(*args): migration = migration_db.load_migration_object(migration_name) start_time = time.time() if action == 'up': if verbosity >= 1: print "Applying migration %s" % migration.name if not options.get('print_plan'): migration_state.apply(migration_name) else: if verbosity >= 1: print "Unapplying migration %s" % migration.name if not options.get('print_plan'): migration_state.unapply(migration_name) if options.get('print_time'): print "Migration %s ran %.1f seconds" % (migration.name, time.time() - start_time) elif args[0] == 'mark_as_applied': migration_state.init() for name in args[1:]: resolved_name = migration_state.resolve_name(name) if resolved_name == None: raise NoSuchMigrationError(name) migration_state.mark_as_applied(resolved_name) elif args[0] == 'mark_as_unapplied': migration_state.init() for name in args[1:]: resolved_name = migration_state.resolve_name(name) if resolved_name == None: raise NoSuchMigrationError(name) migration_state.mark_as_unapplied(resolved_name) elif args[0] == 'list': migration_state.init() for migration_name in migration_db.list(): if migration_state.is_applied(migration_name, use_cache=True): print "* [+] %s" % migration_name else: print "* [ ] %s" % migration_name migrations_not_in_db = migration_state.applied_but_not_in_db() if migrations_not_in_db: print "These migrations are marked as applied but cannot " \ "be found:" for migration_name in migrations_not_in_db: print "* [?] %s" % migration_name return elif args[0] == 'init': migration_state.init() elif args[0] == 'cat': for name in args[1:]: print open( migration_db.resolve_migration_path(name), 'r' ).read() return else: raise CommandError( 'Argument should be one of: list, help, up, down, all, all_hard, init, ' 'apply, unapply, to, downto, upto, mark_as_applied, ' 'mark_as_unapplied' ) # Ensure Django permissions and content_types have been created # NOTE: Don't run if django_content_type doesn't exist yet. if table_present('django_content_type'): from django.contrib.auth.management import create_permissions from django.db import models for app in models.get_apps(): if verbosity >= 1: create_permissions(app, set(), 2) else: create_permissions(app, set(), 1)
def get_modelclasses(): for app in get_apps(): modelclasses = get_models(app) for modelclass in modelclasses: yield modelclass
def update_all_contenttypes(verbosity=2): for app in get_apps(): update_contenttypes(app, None, verbosity)
def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[], suite=None): """ This module allows users to run tests for GIS apps that require the creation of a spatial database. Currently, this is only required for PostgreSQL as PostGIS needs extra overhead in test database creation. In order to create a PostGIS database, the DATABASE_USER (or TEST_DATABASE_USER, if defined) will require superuser priviliges. To accomplish this outside the `postgres` user, you have a few options: (A) Make your user a super user: This may be done at the time the user is created, for example: $ createuser --superuser <user_name> Or you may alter the user's role from the SQL shell (assuming this is done from an existing superuser role): postgres# ALTER ROLE <user_name> SUPERUSER; (B) Create your own PostgreSQL database as a local user: 1. Initialize database: `initdb -D /path/to/user/db` 2. If there's already a Postgres instance on the machine, it will need to use a different TCP port than 5432. Edit postgresql.conf (in /path/to/user/db) to change the database port (e.g. `port = 5433`). 3. Start this database `pg_ctl -D /path/to/user/db start` (C) On Windows platforms the pgAdmin III utility may also be used as a simple way to add superuser privileges to your database user. The TEST_RUNNER needs to be set in your settings like so: TEST_RUNNER='django.contrib.gis.tests.run_tests' Note: This test runner assumes that the PostGIS SQL files ('lwpostgis.sql' and 'spatial_ref_sys.sql') are installed in the directory specified by `pg_config --sharedir` (and defaults to /usr/local/share if that fails). This behavior is overridden if POSTGIS_SQL_PATH is set in your settings. Windows users should set POSTGIS_SQL_PATH manually because the output of `pg_config` uses paths like 'C:/PROGRA~1/POSTGR~1/..'. Finally, the tests may be run by invoking `./manage.py test`. """ # The `create_spatial_db` routine abstracts away all the steps needed # to properly construct a spatial database for the backend. from django.contrib.gis.db.backend import create_spatial_db # Setting up for testing. setup_test_environment() settings.DEBUG = False old_name = settings.DATABASE_NAME # The suite may be passed in manually, e.g., when we run the GeoDjango test, # we want to build it and pass it in due to some customizations. Otherwise, # the normal test suite creation process from `django.test.simple.run_tests` # is used to create the test suite. if suite is None: suite = unittest.TestSuite() if test_labels: for label in test_labels: if '.' in label: suite.addTest(build_test(label)) else: app = get_app(label) suite.addTest(build_suite(app)) else: for app in get_apps(): suite.addTest(build_suite(app)) for test in extra_tests: suite.addTest(test) # Creating the test spatial database. create_spatial_db(test=True, verbosity=verbosity) # Executing the tests (including the model tests), and destorying the # test database after the tests have completed. result = unittest.TextTestRunner(verbosity=verbosity).run(suite) connection.creation.destroy_test_db(old_name, verbosity) teardown_test_environment() # Returning the total failures and errors return len(result.failures) + len(result.errors)
def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]): """ worsk exactly as per normal test but only creates the test_db if it doesn't yet exist and does not destroy it when done tables are flushed and fixtures loaded between tests as per usual but if your schema has not changed then this saves significant amounts of time and speeds up the test cycle Run the unit tests for all the test labels in the provided list. Labels must be of the form: - app.TestClass.test_method Run a single specific test method - app.TestClass Run all the test methods in a given class - app Search for doctests and unittests in the named application. When looking for tests, the test runner will look in the models and tests modules for the application. A list of 'extra' tests may also be provided; these tests will be added to the test suite. Returns the number of tests that failed. """ setup_test_environment() settings.DEBUG = False suite = unittest.TestSuite() if test_labels: for label in test_labels: if '.' in label: suite.addTest(build_test(label)) else: app = get_app(label) suite.addTest(build_suite(app)) else: for app in get_apps(): suite.addTest(build_suite(app)) for test in extra_tests: suite.addTest(test) suite = reorder_suite(suite, (TestCase, )) old_name = settings.DATABASES['default']['NAME'] ###Everything up to here is from django.test.simple from django.db.backends import creation from django.db import connection, DatabaseError if settings.DATABASES['default']['TEST_NAME']: settings.DATABASES['default']['NAME'] = settings.DATABASES['default'][ 'TEST_NAME'] else: settings.DATABASES['default'][ 'NAME'] = creation.TEST_DATABASE_PREFIX + settings.DATABASES[ 'default']['NAME'] connection.settings_dict["DATABASE_NAME"] = settings.DATABASES['default'][ 'NAME'] # does test db exist already ? try: if settings.DATABASES['default']['ENGINE'] == 'sqlite3': if not os.path.exists(settings.DATABASES['default']['NAME']): raise DatabaseError cursor = connection.cursor() except Exception: # db does not exist # juggling ! create_test_db switches the DATABASE_NAME to the TEST_DATABASE_NAME settings.DATABASES['default']['NAME'] = old_name connection.settings_dict["DATABASE_NAME"] = old_name connection.creation.create_test_db(verbosity, autoclobber=True) else: connection.close() settings.DATABASES['default'][ 'SUPPORTS_TRANSACTIONS'] = connections_support_transactions() result = unittest.TextTestRunner(verbosity=verbosity).run(suite) #Since we don't call destory_test_db, we need to set the db name back. settings.DATABASES['default']['NAME'] = old_name connection.settings_dict["DATABASE_NAME"] = old_name teardown_test_environment() return len(result.failures) + len(result.errors)
def sync_apps(self, connection, apps): "Runs the old syncdb-style operation on a list of apps." cursor = connection.cursor() # Get a list of already installed *models* so that references work right. tables = connection.introspection.table_names() seen_models = connection.introspection.installed_models(tables) created_models = set() pending_references = {} # Build the manifest of apps and models that are to be synchronized all_models = [(app.__name__.split('.')[-2], router.get_migratable_models(app, connection.alias, include_auto_created=True)) for app in models.get_apps() if app.__name__.split('.')[-2] in apps] def model_installed(model): opts = model._meta converter = connection.introspection.table_name_converter # Note that if a model is unmanaged we short-circuit and never try to install it return not ((converter(opts.db_table) in tables) or (opts.auto_created and converter( opts.auto_created._meta.db_table) in tables)) manifest = OrderedDict( (app_name, list(filter(model_installed, model_list))) for app_name, model_list in all_models) create_models = set(itertools.chain(*manifest.values())) emit_pre_migrate_signal(create_models, self.verbosity, self.interactive, connection.alias) # Create the tables for each model if self.verbosity >= 1: self.stdout.write(" Creating tables...\n") with transaction.atomic(using=connection.alias, savepoint=False): for app_name, model_list in manifest.items(): for model in model_list: # Create the model's database table, if it doesn't already exist. if self.verbosity >= 3: self.stdout.write(" Processing %s.%s model\n" % (app_name, model._meta.object_name)) sql, references = connection.creation.sql_create_model( model, no_style(), seen_models) seen_models.add(model) created_models.add(model) for refto, refs in references.items(): pending_references.setdefault(refto, []).extend(refs) if refto in seen_models: sql.extend( connection.creation.sql_for_pending_references( refto, no_style(), pending_references)) sql.extend( connection.creation.sql_for_pending_references( model, no_style(), pending_references)) if self.verbosity >= 1 and sql: self.stdout.write(" Creating table %s\n" % model._meta.db_table) for statement in sql: cursor.execute(statement) tables.append( connection.introspection.table_name_converter( model._meta.db_table)) # We force a commit here, as that was the previous behaviour. # If you can prove we don't need this, remove it. transaction.set_dirty(using=connection.alias) # The connection may have been closed by a syncdb handler. cursor = connection.cursor() # Install custom SQL for the app (but only if this # is a model we've just created) if self.verbosity >= 1: self.stdout.write(" Installing custom SQL...\n") for app_name, model_list in manifest.items(): for model in model_list: if model in created_models: custom_sql = custom_sql_for_model(model, no_style(), connection) if custom_sql: if self.verbosity >= 2: self.stdout.write( " Installing custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) try: with transaction.commit_on_success_unless_managed( using=connection.alias): for sql in custom_sql: cursor.execute(sql) except Exception as e: self.stderr.write( " Failed to install custom SQL for %s.%s model: %s\n" % (app_name, model._meta.object_name, e)) if self.show_traceback: traceback.print_exc() else: if self.verbosity >= 3: self.stdout.write( " No custom SQL for %s.%s model\n" % (app_name, model._meta.object_name)) if self.verbosity >= 1: self.stdout.write(" Installing indexes...\n") # Install SQL indices for all newly created models for app_name, model_list in manifest.items(): for model in model_list: if model in created_models: index_sql = connection.creation.sql_indexes_for_model( model, no_style()) if index_sql: if self.verbosity >= 2: self.stdout.write( " Installing index for %s.%s model\n" % (app_name, model._meta.object_name)) try: with transaction.commit_on_success_unless_managed( using=connection.alias): for sql in index_sql: cursor.execute(sql) except Exception as e: self.stderr.write( " Failed to install index for %s.%s model: %s\n" % (app_name, model._meta.object_name, e)) # Load initial_data fixtures (unless that has been disabled) if self.load_initial_data: call_command('loaddata', 'initial_data', verbosity=self.verbosity, database=connection.alias, skip_validation=True) return created_models
#!/usr/bin/env python from django.core.management import setup_environ try: import settings except ImportError: import sys sys.stderr.write("Couldn't find the settings.py module.") sys.exit(1) setup_environ(settings) # Add any missing content types from django.contrib.contenttypes.management import update_all_contenttypes update_all_contenttypes() # Add any missing permissions from django.contrib.auth.management import create_permissions from django.db.models import get_apps for app in get_apps(): create_permissions(app, None, 2) print('done!')
def handle(self, *fixture_labels, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len( self.namelist() ) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [ os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity >= 2: self.stdout.write("Loading '%s' fixtures...\n" % fixture_name) else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s is not a known serialization format.\n" % (fixture_name, format))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity >= 2: self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join( p for p in [fixture_name, database, format, compression_format] if p) if verbosity >= 3: self.stdout.write("Trying %s for %s fixture '%s'...\n" % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') if label_found: fixture.close() self.stderr.write( self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting.\n" % (fixture_name, humanize(fixture_dir)))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return else: fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if verbosity >= 2: self.stdout.write("Installing %s fixture '%s' from %s.\n" % \ (format, fixture_name, humanize(fixture_dir))) try: objects = serializers.deserialize(format, fixture, using=using) with connection.constraint_checks_disabled(): for obj in objects: objects_in_fixture += 1 if router.allow_syncdb( using, obj.object.__class__): loaded_objects_in_fixture += 1 models.add(obj.object.__class__) obj.save(using=using) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [ model._meta.db_table for model in models ] connection.check_constraints( table_names=table_names) loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) if show_traceback: traceback.print_exc() else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, ''.join( traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))))) return fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: self.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)\n" % (fixture_name))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return except Exception, e: if verbosity >= 2: self.stdout.write("No %s fixture '%s' in %s.\n" % \ (format, fixture_name, humanize(fixture_dir)))
def handle_noargs(self, **options): from django.contrib.auth.management import create_permissions from django.db.models import get_apps for app in get_apps(): create_permissions(app, None, 2)
def model_from_db_table(db_table): for app in models.get_apps(): for model in models.get_models(app): if model._meta.db_table == db_table: return model raise ValueError("Couldn't find model class for %s" % db_table)
def generate_dot(app_labels, **kwargs): disable_fields = kwargs.get('disable_fields', False) include_models = parse_file_or_list(kwargs.get('include_models', "")) all_applications = kwargs.get('all_applications', False) use_subgraph = kwargs.get('group_models', False) verbose_names = kwargs.get('verbose_names', False) inheritance = kwargs.get('inheritance', False) language = kwargs.get('language', None) if language is not None: activate_language(language) exclude_columns = parse_file_or_list(kwargs.get('exclude_columns', "")) exclude_models = parse_file_or_list(kwargs.get('exclude_models', "")) def skip_field(field): if exclude_columns: if verbose_names and field.verbose_name: if field.verbose_name in exclude_columns: return True if field.name in exclude_columns: return True return False t = loader.get_template_from_string(""" digraph name { fontname = "Helvetica" fontsize = 8 node [ fontname = "Helvetica" fontsize = 8 shape = "plaintext" ] edge [ fontname = "Helvetica" fontsize = 8 ] """) c = Context({}) dot = t.render(c) apps = [] if all_applications: apps = models.get_apps() for app_label in app_labels: app = models.get_app(app_label) if not app in apps: apps.append(app) graphs = [] for app in apps: graph = Context({ 'name': '"%s"' % app.__name__, 'app_name': "%s" % '.'.join(app.__name__.split('.')[:-1]), 'cluster_app_name': "cluster_%s" % app.__name__.replace(".", "_"), 'disable_fields': disable_fields, 'use_subgraph': use_subgraph, 'models': [] }) appmodels = get_models(app) abstract_models = [] for appmodel in appmodels: abstract_models = abstract_models + [ abstract_model for abstract_model in appmodel.__bases__ if hasattr( abstract_model, '_meta') and abstract_model._meta.abstract ] abstract_models = list(set(abstract_models)) # remove duplicates appmodels = abstract_models + appmodels for appmodel in appmodels: appmodel_abstracts = [ abstract_model.__name__ for abstract_model in appmodel.__bases__ if hasattr(abstract_model, '_meta') and abstract_model._meta.abstract ] # collect all attribs of abstract superclasses def getBasesAbstractFields(c): _abstract_fields = [] for e in c.__bases__: if hasattr(e, '_meta') and e._meta.abstract: _abstract_fields.extend(e._meta.fields) _abstract_fields.extend(getBasesAbstractFields(e)) return _abstract_fields abstract_fields = getBasesAbstractFields(appmodel) model = { 'app_name': appmodel.__module__.replace(".", "_"), 'name': appmodel.__name__, 'abstracts': appmodel_abstracts, 'fields': [], 'relations': [] } # consider given model name ? def consider(model_name): if exclude_models and model_name in exclude_models: return False return not include_models or model_name in include_models if not consider(appmodel._meta.object_name): continue if verbose_names and appmodel._meta.verbose_name: model['label'] = appmodel._meta.verbose_name else: model['label'] = model['name'] # model attributes def add_attributes(field): if verbose_names and field.verbose_name: label = field.verbose_name else: label = field.name t = type(field).__name__ if isinstance(field, (OneToOneField, ForeignKey)): t += " ({0})".format(field.rel.field_name) # TODO: ManyToManyField, GenericRelation model['fields'].append({ 'name': field.name, 'label': label, 'type': t, 'blank': field.blank, 'abstract': field in abstract_fields, }) # Find all the real attributes. Relations are depicted as graph edges instead of attributes attributes = [ field for field in appmodel._meta.local_fields if not isinstance(field, RelatedField) ] # find primary key and print it first, ignoring implicit id if other pk exists pk = appmodel._meta.pk if not appmodel._meta.abstract and pk in attributes: add_attributes(pk) for field in attributes: if skip_field(field): continue if not field.primary_key: add_attributes(field) # FIXME: actually many_to_many fields aren't saved in this model's db table, so why should we add an attribute-line for them in the resulting graph? #if appmodel._meta.many_to_many: # for field in appmodel._meta.many_to_many: # if skip_field(field): # continue # add_attributes(field) # relations def add_relation(field, extras=""): if verbose_names and field.verbose_name: label = field.verbose_name else: label = field.name # show related field name if hasattr(field, 'related_query_name'): label += ' (%s)' % field.related_query_name() # handle self-relationships if field.rel.to == 'self': target_model = field.model else: target_model = field.rel.to _rel = { 'target_app': target_model.__module__.replace('.', '_'), 'target': target_model.__name__, 'type': type(field).__name__, 'name': field.name, 'label': label, 'arrows': extras, 'needs_node': True } if _rel not in model['relations'] and consider(_rel['target']): model['relations'].append(_rel) for field in appmodel._meta.local_fields: if field.attname.endswith( '_ptr_id' ): # excluding field redundant with inheritance relation continue if field in abstract_fields: # excluding fields inherited from abstract classes. they too show as local_fields continue if skip_field(field): continue if isinstance(field, OneToOneField): add_relation(field, '[arrowhead=none, arrowtail=none]') elif isinstance(field, ForeignKey): add_relation(field, '[arrowhead=none, arrowtail=dot]') for field in appmodel._meta.local_many_to_many: if skip_field(field): continue if isinstance(field, ManyToManyField): if (getattr(field, 'creates_table', False) or # django 1.1. (hasattr(field.rel.through, '_meta') and field.rel.through._meta.auto_created)): # django 1.2 add_relation( field, '[arrowhead=dot arrowtail=dot, dir=both]') elif isinstance(field, GenericRelation): add_relation( field, mark_safe( '[style="dotted", arrowhead=normal, arrowtail=normal, dir=both]' )) if inheritance: # add inheritance arrows for parent in appmodel.__bases__: if hasattr(parent, "_meta"): # parent is a model l = "multi-table" if parent._meta.abstract: l = "abstract" if appmodel._meta.proxy: l = "proxy" l += r"\ninheritance" _rel = { 'target_app': parent.__module__.replace(".", "_"), 'target': parent.__name__, 'type': "inheritance", 'name': "inheritance", 'label': l, 'arrows': '[arrowhead=empty, arrowtail=none]', 'needs_node': True } # TODO: seems as if abstract models aren't part of models.getModels, which is why they are printed by this without any attributes. if _rel not in model['relations'] and consider( _rel['target']): model['relations'].append(_rel) graph['models'].append(model) graphs.append(graph) nodes = [] for graph in graphs: nodes.extend([e['name'] for e in graph['models']]) for graph in graphs: # don't draw duplication nodes because of relations for model in graph['models']: for relation in model['relations']: if relation['target'] in nodes: relation['needs_node'] = False # render templates t = loader.get_template_from_string("""{% if use_subgraph %} subgraph {{ cluster_app_name }} { label=< <TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0"> <TR><TD COLSPAN="2" CELLPADDING="4" ALIGN="CENTER" ><FONT FACE="Helvetica Bold" COLOR="Black" POINT-SIZE="12" >{{ app_name }}</FONT></TD></TR> </TABLE> > color=olivedrab4 style="rounded" {% endif %} {% for model in models %} {{ model.app_name }}_{{ model.name }} [label=< <TABLE BGCOLOR="palegoldenrod" BORDER="0" CELLBORDER="0" CELLSPACING="0"> <TR><TD COLSPAN="2" CELLPADDING="4" ALIGN="CENTER" BGCOLOR="olivedrab4" ><FONT FACE="Helvetica Bold" COLOR="white" >{{ model.label }}{% if model.abstracts %}<BR/><<FONT FACE="Helvetica Italic">{{ model.abstracts|join:"," }}</FONT>>{% endif %}</FONT></TD></TR> {% if not disable_fields %} {% for field in model.fields %} <TR><TD ALIGN="LEFT" BORDER="0" ><FONT {% if field.blank %}COLOR="#7B7B7B" {% endif %}FACE="Helvetica {% if field.abstract %}Italic{% else %}Bold{% endif %}">{{ field.label }}</FONT ></TD> <TD ALIGN="LEFT" ><FONT {% if field.blank %}COLOR="#7B7B7B" {% endif %}FACE="Helvetica {% if field.abstract %}Italic{% else %}Bold{% endif %}">{{ field.type }}</FONT ></TD></TR> {% endfor %} {% endif %} </TABLE> >] {% endfor %} {% if use_subgraph %} } {% endif %}""") dot += '\n' + t.render(graph) for graph in graphs: t = loader.get_template_from_string("""{% for model in models %} {% for relation in model.relations %} {% if relation.needs_node %} {{ relation.target_app }}_{{ relation.target }} [label=< <TABLE BGCOLOR="palegoldenrod" BORDER="0" CELLBORDER="0" CELLSPACING="0"> <TR><TD COLSPAN="2" CELLPADDING="4" ALIGN="CENTER" BGCOLOR="olivedrab4" ><FONT FACE="Helvetica Bold" COLOR="white" >{{ relation.target }}</FONT></TD></TR> </TABLE> >] {% endif %} {{ model.app_name }}_{{ model.name }} -> {{ relation.target_app }}_{{ relation.target }} [label="{{ relation.label }}"] {{ relation.arrows }}; {% endfor %} {% endfor %}""") dot += '\n' + t.render(graph) t = loader.get_template_from_string("}") c = Context({}) dot += '\n' + t.render(c) return dot
def load_initial_objects(self): # look for any .xml files in apps under fixtures/initial_objects # and attempt to load them as Fedora objects # NOTE! any fixtures should have pids specified, or new versions of the # fixture will be created every time syncrepo runs app_module_paths = [] if hasattr(django_apps, 'get_app_configs'): apps = django_apps.get_app_configs() else: apps = get_apps() # monkey see django code, monkey do for app in apps: # newer django AppConfig if hasattr(app, 'path'): app_module_paths.append(app.path) elif hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixture_paths = [ os.path.join(os.path.dirname(path), 'fixtures', 'initial_objects', '*.xml') for path in app_module_paths ] fixture_count = 0 load_count = 0 for path in app_fixture_paths: fixtures = glob.iglob(path) for f in fixtures: # FIXME: is there a sane, sensible way to shorten file path for error/success messages? fixture_count += 1 with open(f) as fixture_data: # rather than pulling PID from fixture and checking if it already exists, # just ingest and catch appropriate excetions try: pid = self.repo.ingest(fixture_data.read(), "loaded from fixture") if self.verbosity > 1: self.stdout.write("Loaded fixture %s as %s" % (f, pid)) load_count += 1 except RequestFailed as rf: if hasattr(rf, 'detail'): if 'ObjectExistsException' in rf.detail or \ 'already exists in the registry; the object can\'t be re-created' in rf.detail: if self.verbosity > 1: self.stdout.write( "Fixture %s has already been loaded" % f) elif 'ObjectValidityException' in rf.detail: # could also look for: fedora.server.errors.ValidationException # (e.g., RELS-EXT about does not match pid) self.stdout.write( "Error: fixture %s is not a valid Repository object" % f) else: # if there is at least a detail message, display that self.stdout.write("Error ingesting %s: %s" % (f, rf.detail)) else: raise rf # summarize what was actually done if self.verbosity > 0: if fixture_count == 0: self.stdout.write("No fixtures found") else: self.stdout.write("Loaded %d object(s) from %d fixture(s)" % (load_count, fixture_count))
def Deserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ db = options.pop('using', DEFAULT_DB_ALIAS) models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get( "encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__'): return field.rel.to._default_manager.db_manager( db).get_by_natural_key(*value).pk else: return smart_unicode( field.rel.to._meta.pk.to_python(value)) else: m2m_convert = lambda v: smart_unicode( field.rel.to._meta.pk.to_python(v)) m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): if hasattr(field_value, '__iter__'): obj = field.rel.to._default_manager.db_manager( db).get_by_natural_key(*field_value) value = getattr(obj, field.rel.field_name) # If this is a natural foreign key to an object that # has a FK/O2O as the foreign key, use the FK value if field.rel.to._meta.pk.rel: value = value.pk else: value = field.rel.to._meta.get_field( field.rel.field_name).to_python(field_value) data[field.attname] = value else: data[field.attname] = field.rel.to._meta.get_field( field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def handle_noargs(self, **options): db = options.get('database') connection = connections[db] verbosity = int(options.get('verbosity')) interactive = options.get('interactive') self.style = no_style() # Import the 'management' module within each installed app, to register # dispatcher events. for app_name in settings.INSTALLED_APPS: try: import_module('.management', app_name) except ImportError: pass sql_list = sql_flush(self.style, connection, only_django=True) if interactive: confirm = raw_input("""You have requested a flush of the database. This will IRREVERSIBLY DESTROY all data currently in the %r database, and return each table to the state it was in after syncdb. Are you sure you want to do this? Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME']) else: confirm = 'yes' if confirm == 'yes': try: cursor = connection.cursor() for sql in sql_list: cursor.execute(sql) except Exception as e: transaction.rollback_unless_managed(using=db) raise CommandError( """Database %s couldn't be flushed. Possible reasons: * The database isn't running or isn't configured correctly. * At least one of the expected database tables doesn't exist. * The SQL was invalid. Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run. The full error: %s""" % (connection.settings_dict['NAME'], e)) transaction.commit_unless_managed(using=db) # Emit the post sync signal. This allows individual # applications to respond as if the database had been # sync'd from scratch. all_models = [] for app in models.get_apps(): all_models.extend([ m for m in models.get_models(app, include_auto_created=True) if router.allow_syncdb(db, m) ]) emit_post_sync_signal(set(all_models), verbosity, interactive, db) # Reinstall the initial_data fixture. kwargs = options.copy() kwargs['database'] = db if options.get('load_initial_data', True): # Reinstall the initial_data fixture. from django.core.management import call_command call_command('loaddata', 'initial_data', **options) else: self.stdout.write("Flush cancelled.\n")
def handle(self, *fixture_labels, **options): using = options.get('database') connection = connections[using] if not len(fixture_labels): raise CommandError( "No database fixture specified. Please provide the path of at " "least one fixture in the command line.") verbosity = int(options.get('verbosity')) show_traceback = options.get('traceback') # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 models = set() humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len( self.namelist() ) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [ os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths ] try: with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats( ): formats = [format] else: formats = [] if formats: if verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) else: raise CommandError( "Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format)) if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join(p for p in [ fixture_name, database, format, compression_format ] if p) if verbosity >= 3: self.stdout.write("Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') except IOError: if verbosity >= 2: self.stdout.write("No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir))) else: try: if label_found: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if verbosity >= 2: self.stdout.write("Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize( format, fixture, using=using) for obj in objects: objects_in_fixture += 1 if router.allow_syncdb( using, obj.object.__class__): loaded_objects_in_fixture += 1 models.add(obj.object.__class__) try: obj.save(using=using) except (DatabaseError, IntegrityError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta. app_label, 'object_name': obj.object._meta. object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) }, ) raise loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True except Exception as e: if not isinstance(e, CommandError): e.args = ( "Problem installing fixture '%s': %s" % (full_path, e), ) raise finally: fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: raise CommandError( "No fixture data found for '%s'. (File format may be invalid.)" % (fixture_name)) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e, ) raise except (SystemExit, KeyboardInterrupt): raise except Exception as e: if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) raise # If we found even one object in a fixture, we need to reset the # database sequences. if loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( no_style(), models) if sequence_sql: if verbosity >= 2: self.stdout.write("Resetting sequences\n") for line in sequence_sql: cursor.execute(line) if commit: transaction.commit(using=using) transaction.leave_transaction_management(using=using) if verbosity >= 1: if fixture_object_count == loaded_object_count: self.stdout.write("Installed %d object(s) from %d fixture(s)" % (loaded_object_count, fixture_count)) else: self.stdout.write( "Installed %d object(s) (of %d) from %d fixture(s)" % (loaded_object_count, fixture_object_count, fixture_count)) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. if commit: connection.close()