def get_objects(): for model in serializers.sort_dependencies(app_list): if (model._meta.can_migrate(self.connection) and router.allow_migrate_model(self.connection.alias, model)): queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name) for obj in queryset.iterator(): yield obj
def create_table(self, database, tablename, dry_run): cache = BaseDatabaseCache(tablename, {}) if not router.allow_migrate_model(database, cache.cache_model_class): return connection = connections[database] if tablename in connection.introspection.table_names(): if self.verbosity > 0: self.stdout.write("Cache table '%s' already exists." % tablename) return fields = ( # "key" is a reserved word in MySQL, so use "cache_key" instead. models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True), models.TextField(name='value'), models.DateTimeField(name='expires', db_index=True), ) table_output = [] index_output = [] qn = connection.ops.quote_name for f in fields: field_output = [qn(f.name), f.db_type(connection=connection)] field_output.append("%sNULL" % ("NOT " if not f.null else "")) if f.primary_key: field_output.append("PRIMARY KEY") elif f.unique: field_output.append("UNIQUE") if f.db_index: unique = "UNIQUE " if f.unique else "" index_output.append("CREATE %sINDEX %s ON %s (%s);" % (unique, qn('%s_%s' % (tablename, f.name)), qn(tablename), qn(f.name))) table_output.append(" ".join(field_output)) full_statement = ["CREATE TABLE %s (" % qn(tablename)] for i, line in enumerate(table_output): full_statement.append(' %s%s' % (line, ',' if i < len(table_output) - 1 else '')) full_statement.append(');') full_statement = "\n".join(full_statement) if dry_run: self.stdout.write(full_statement) for statement in index_output: self.stdout.write(statement) return with transaction.atomic(using=database, savepoint=connection.features.can_rollback_ddl): with connection.cursor() as curs: try: curs.execute(full_statement) except DatabaseError as e: raise CommandError( "Cache table '%s' could not be created.\nThe error was: %s." % (tablename, force_text(e))) for statement in index_output: curs.execute(statement) if self.verbosity > 1: self.stdout.write("Cache table '%s' created." % tablename)
def db_router_allows_migrate(database, app_label, model_cls): """Return whether a database router allows migrate operations for a model. This will only return ``True`` for Django 1.7 and newer and if the router allows migrate operations. This is compatible with both the Django 1.7 and 1.8+ versions of ``allow_migrate``. Args: database (unicode): The name of the database. app_label (unicode): The application label. model_cls (type): The model class. Returns: bool: ``True`` if routers allow migrate for this model. """ if django.VERSION[:2] >= (1, 8): return router.allow_migrate_model(database, model_cls) elif django.VERSION[:2] == (1, 7): return router.allow_migrate(database, model_cls) else: return False
def save_all(objects, using, iterations=0, stdout=None): '''Tries recursively saves all objects''' if len(objects) == 0 or iterations >= 25: return iterations skiped = [] for obj in objects: if router.allow_migrate_model(using, obj.object.__class__): try: obj.save(using=using) except (DatabaseError, IntegrityError) as e: e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) },) # raise skiped.append(obj) if len(skiped) > 0: return save_all(skiped, using, iterations + 1) return iterations
def load_data_for_db(db_alias, objects): """ :param db_alias: Django alias for database to load objects into :param objects: List of object dictionaries to load :return: LoadStats object """ connection = connections[db_alias] model_counter = Counter() with connection.constraint_checks_disabled(): for obj in PythonDeserializer(objects, using=db_alias): if router.allow_migrate_model(db_alias, obj.object.__class__): model_counter.update([obj.object.__class__]) try: obj.save(using=db_alias) except (DatabaseError, IntegrityError) as e: e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) },) raise # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in model_counter] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem loading data: %s" % e,) raise return LoadStat(db_alias, model_counter)
def get_objects(count_only=False): """ Collate the objects to be serialized. If count_only is True, just count the number of objects to be serialized. """ models = serializers.sort_dependencies(app_list.items()) for model in models: if model in excluded_models: continue if model._meta.proxy and model._meta.proxy_for_model not in models: warnings.warn( "%s is a proxy model and won't be serialized." % model._meta.label, category=ProxyModelWarning, ) if not model._meta.proxy and router.allow_migrate_model(using, model): if use_base_manager: objects = model._base_manager else: objects = model._default_manager queryset = objects.using(using).order_by(model._meta.pk.name) if primary_keys: queryset = queryset.filter(pk__in=primary_keys) if count_only: yield queryset.order_by().count() else: for obj in queryset.iterator(): yield obj
def create_default_site(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): try: Site = apps.get_model('sites', 'Site') except LookupError: return if not router.allow_migrate_model(using, Site): return if not Site.objects.using(using).exists(): # The default settings set SITE_ID = 1, and some tests in Django's test # suite rely on this value. However, if database sequences are reused # (e.g. in the test suite after flush/syncdb), it isn't guaranteed that # the next id will be 1, so we coerce it. See #15573 and #16353. This # can also crop up outside of tests - see #15346. if verbosity >= 2: print("Creating example.com Site object") Site(pk=settings.SITE_ID, domain="example.com", name="example.com").save(using=using) # We set an explicit pk instead of relying on auto-incrementation, # so we need to reset the database sequence. See #17415. sequence_sql = connections[using].ops.sequence_reset_sql(no_style(), [Site]) if sequence_sql: if verbosity >= 2: print("Resetting sequence") with connections[using].cursor() as cursor: for command in sequence_sql: cursor.execute(command)
def load_cloud_objects(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): if not app_config.models_module: return initial_data = load_initial_data() for model in initial_data: logger.info("Attempting to load data for {}...".format(model['model'])) # Grab the model class, but don't fail if we can't find it try: model_cls = apps.get_model('cloud', model['model']) except LookupError: logger.warning('Failed to load model class: {}'.format(model['model'])) continue # If we can't migrate this model, don't do anything & go on to the next if not router.allow_migrate_model(using, model_cls): logger.info("Skipping data load for {}".format(model['model'])) continue to_create = [] # Grab the attribute to filter on filter_attr = model['filter_attr'] for object_data in model['objects']: # Only create if it's not already there filtered = model_cls.objects.filter(**{filter_attr: object_data[filter_attr]}) if filtered.count() == 0: # Object doesn't exist in the database if 'pk' in object_data: try: model_cls.objects.get(pk=object_data['pk']) # This means there's a conflicting pk, so we'll just remove the pk from # the object_data dict and let it get assigned something else. del object_data['pk'] except exceptions.ObjectDoesNotExist: # There's no conflicting object, this is good pass # Add this object to the create list to_create.append(model_cls(**object_data)) else: # This object already exists in the database # Update the fields to match the new stuff if the object already exists for obj in filtered: for attr, val in object_data.items(): # We don't want to change the object's primary key # it would mess up a bunch of relations if attr != 'pk': setattr(obj, attr, val) obj.save() # bulk create everything model_cls.objects.using(using).bulk_create(to_create)
def load_label(self, fixture_label): """ Loads fixtures files for a given label. """ show_progress = self.verbosity >= 3 for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label): _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file)) open_method, mode = self.compression_formats[cmp_fmt] fixture = open_method(fixture_file, mode) try: self.fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if self.verbosity >= 2: self.stdout.write("Installing %s fixture '%s' from %s." % (ser_fmt, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize(ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore) for obj in objects: objects_in_fixture += 1 if router.allow_migrate_model(self.using, obj.object.__class__): loaded_objects_in_fixture += 1 self.models.add(obj.object.__class__) try: obj.save(using=self.using) if show_progress: self.stdout.write( '\rProcessed %i object(s).' % loaded_objects_in_fixture, ending='' ) except (DatabaseError, IntegrityError) as e: e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) },) raise if objects and show_progress: self.stdout.write('') # add a newline after progress indicator self.loaded_object_count += loaded_objects_in_fixture self.fixture_object_count += objects_in_fixture except Exception as e: if not isinstance(e, CommandError): e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),) raise finally: fixture.close() # Warn if the fixture we loaded contains 0 objects. if objects_in_fixture == 0: warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, RuntimeWarning )
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): if not app_config.models_module: return try: Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return from django.contrib.contenttypes.models import ContentType # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() for klass in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta, ctype): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set(Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list( "content_type", "codename" )) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] # Validate the permissions before bulk_creation to avoid cryptic # database error when the verbose_name is longer than 50 characters permission_name_max_length = Permission._meta.get_field('name').max_length verbose_name_max_length = permission_name_max_length - 11 # len('Can change ') prefix for perm in perms: if len(perm.name) > permission_name_max_length: raise exceptions.ValidationError( "The verbose_name of %s.%s is longer than %s characters" % ( perm.content_type.app_label, perm.content_type.model, verbose_name_max_length, ) ) Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def allow_migrate_model(self, connection_alias, model): """ Returns if we're allowed to migrate the model. This is a thin wrapper around router.allow_migrate_model() that preemptively rejects any proxy, swapped out, or unmanaged model. """ if not model._meta.can_migrate(connection_alias): return False return router.allow_migrate_model(connection_alias, model)
def load_label(self, fixture_label, ser_fmt='json'): """ Uses get_latest_fixture to load fixture data """ with get_latest_fixture(fixture_label) as fixture: try: self.fixture_name = fixture.name self.fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if self.verbosity >= 2: self.stdout.write("Installing %s fixture." % ( fixture.name )) objects = serializers.deserialize( ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore ) for obj in objects: objects_in_fixture += 1 if router.allow_migrate_model(self.using, obj.object.__class__): loaded_objects_in_fixture += 1 self.models.add(obj.object.__class__) try: obj.save(using=self.using) except (DatabaseError, IntegrityError) as e: e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) },) raise self.loaded_object_count += loaded_objects_in_fixture self.fixture_object_count += objects_in_fixture except Exception as e: e.args = ("Problem installing fixture '%s': %s" % (fixture.name, e),) raise finally: fixture.close() # Warn if the fixture we loaded contains 0 objects. if objects_in_fixture == 0: warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture.name, RuntimeWarning )
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): if not app_config.models_module: return # Ensure that contenttypes are created for this app. Needed if # 'django.contrib.auth' is in INSTALLED_APPS before # 'django.contrib.contenttypes'. create_contenttypes(app_config, verbosity=verbosity, interactive=interactive, using=using, apps=apps, **kwargs) app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = [] # The codenames and ctypes that should exist. ctypes = set() for klass in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(klass, for_concrete_model=False) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set(Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list( "content_type", "codename" )) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def get_contenttypes_and_models(app_config, using, ContentType): if not router.allow_migrate_model(using, ContentType): return None, None ContentType.objects.clear_cache() content_types = { ct.model: ct for ct in ContentType.objects.using(using).filter(app_label=app_config.label) } app_models = { model._meta.model_name: model for model in app_config.get_models() } return content_types, app_models
def get_all_model_querysets_for_domain(model_class, domain): using = router.db_for_read(model_class) if settings.USE_PARTITIONED_DATABASE and using == partition_config.get_proxy_db(): using = partition_config.get_form_processing_dbs() else: using = [using] for db_alias in using: if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class): objects = model_class._default_manager queryset = objects.using(db_alias).order_by(model_class._meta.pk.name) filters = get_model_domain_filters(model_class, domain) for filter in filters: yield model_class, queryset.filter(filter)
def get_objects(): # Collate the objects to be serialized. for model in serializers.sort_dependencies(app_list.items()): if model in excluded_models: continue if not model._meta.proxy and router.allow_migrate_model(using, model): if use_base_manager: objects = model._base_manager else: objects = model._default_manager queryset = objects.using(using).order_by(model._meta.pk.name) if primary_keys: queryset = queryset.filter(pk__in=primary_keys) for obj in queryset.iterator(): yield obj
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None): using = router.db_for_read(model_class) if settings.USE_PARTITIONED_DATABASE and using == partition_config.get_proxy_db(): using = partition_config.get_form_processing_dbs() else: using = [using] if limit_to_db: if limit_to_db not in using: raise DomainDumpError('DB specified is not valide for ' 'model class: {} not in {}'.format(limit_to_db, using)) using = [limit_to_db] for db_alias in using: if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class): iterator_builder = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)] yield model_class, iterator_builder.build(domain, model_class, db_alias)
def create_default_site_relations(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): try: Site = apps.get_model('sites', 'Site') SiteResources = apps.get_model('geosites', 'SiteResources') SitePeople = apps.get_model('geosites', 'SitePeople') SiteGroups = apps.get_model('geosites', 'SiteGroups') except LookupError: return if not router.allow_migrate_model(using, SiteResources): return for site in Site.objects.all(): SiteResources.objects.get_or_create(site=site) SitePeople.objects.get_or_create(site=site) SiteGroups.objects.get_or_create(site=site)
def update_default_site(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): try: site_model = apps.get_model('sites', 'Site') except LookupError: return if not router.allow_migrate_model(using, site_model): return if verbosity >= 2: print('Updating example.com Site object') if getattr(settings, 'SITE_ID', None): site = site_model.objects.get(pk=settings.SITE_ID) site.domain = settings.SITE_DOMAIN site.name = settings.SITE_NAME site.save(using=using)
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): """Create permissions on django 1.10+.""" if not app_config.models_module: return try: ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return searched_perms = list() # (content_type, (codename, name)) ctypes = set() # The codenames and ctypes that should exist. for klass in app_config.get_models(): ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) from django.contrib.auth.management import _get_all_permissions for perm in _get_all_permissions(klass._meta): searched_perms.append((ctype, perm)) all_perms = set(Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list( "content_type", "codename" )) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def inject_rename_contenttypes_operations(plan=None, apps=global_apps, using=DEFAULT_DB_ALIAS, **kwargs): """ Insert a `RenameContentType` operation after every planned `RenameModel` operation. """ if plan is None: return # Determine whether or not the ContentType model is available. try: ContentType = apps.get_model('contenttypes', 'ContentType') except LookupError: available = False else: if not router.allow_migrate_model(using, ContentType): return available = True for migration, backward in plan: if ((migration.app_label, migration.name) == ('contenttypes', '0001_initial')): # There's no point in going forward if the initial contenttypes # migration is unapplied as the ContentType model will be # unavailable from this point. if backward: break else: available = True continue # The ContentType model is not available yet. if not available: continue inserts = [] for index, operation in enumerate(migration.operations): if isinstance(operation, migrations.RenameModel): operation = RenameContentType( migration.app_label, operation.old_name_lower, operation.new_name_lower ) inserts.append((index + 1, operation)) for inserted, (index, operation) in enumerate(inserts): migration.operations.insert(inserted + index, operation)
def _precompile_fixture(self, fixture, db): """Precompile a fixture. The fixture is loaded and deserialized, and the resulting objects are stored for future use. Args: fixture (unicode): The name of the fixture. db (unicode): The database name to load fixture data on. """ assert db in self._precompiled_fixtures assert fixture not in self._precompiled_fixtures[db] fixture_path = None for fixture_dir in self._get_fixture_dirs(): fixture_path = os.path.join(fixture_dir, fixture + '.json') if os.path.exists(fixture_path): break try: if not fixture_path: raise IOError('Fixture path not found') with open(fixture_path, 'r') as fp: self._precompiled_fixtures[db][fixture] = [ obj for obj in serializers.deserialize('json', fp, using=db) if ((hasattr(router, 'allow_syncdb') and router.allow_syncdb(db, obj.object.__class__)) or (hasattr(router, 'allow_migrate_model') and router.allow_migrate_model(db, obj.object))) ] except IOError as e: sys.stderr.write('Unable to load fixture %s: %s\n' % (fixture, e))
def create_app_sig(app, database): """ Creates a dictionary representation of the models in a given app. Only those attributes that are interesting from a schema-evolution perspective are included. """ app_sig = OrderedDict() for model in get_models(app): # Only include those models that can be synced. # # On Django 1.7 and up, we need to check if the model allows for # migrations (using allow_migrate_model). # # On older versions of Django, we check if the model allows for # synchronization to the database (allow_syncdb). if ((hasattr(router, 'allow_syncdb') and router.allow_syncdb(database, model.__class__)) or (hasattr(router, 'allow_migrate_model') and router.allow_migrate_model(database, model))): app_sig[model._meta.object_name] = create_model_sig(model) return app_sig
def _rename(self, apps, schema_editor, old_model, new_model): ContentType = apps.get_model('contenttypes', 'ContentType') db = schema_editor.connection.alias if not router.allow_migrate_model(db, ContentType): return try: content_type = ContentType.objects.db_manager(db).get_by_natural_key(self.app_label, old_model) except ContentType.DoesNotExist: pass else: content_type.model = new_model try: with transaction.atomic(using=db): content_type.save(update_fields={'model'}) except IntegrityError: # Gracefully fallback if a stale content type causes a # conflict as update_contenttypes will take care of asking the # user what should be done next. content_type.model = old_model else: # Clear the cache as the `get_by_natual_key()` call will cache # the renamed ContentType instance by its old model name. ContentType.objects.clear_cache()
def load_label(self, fixture_label): """Load fixtures files for a given label.""" show_progress = self.verbosity >= 3 for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label): _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file)) open_method, mode = self.compression_formats[cmp_fmt] fixture = open_method(fixture_file, mode) try: self.fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if self.verbosity >= 2: self.stdout.write( "Installing %s fixture '%s' from %s." % (ser_fmt, fixture_name, humanize(fixture_dir)) ) objects = serializers.deserialize( ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore, ) for obj in objects: objects_in_fixture += 1 if (obj.object._meta.app_config in self.excluded_apps or type(obj.object) in self.excluded_models): continue if router.allow_migrate_model(self.using, obj.object.__class__): loaded_objects_in_fixture += 1 model = obj.object.__class__ self.models.add(model) try: obj.object.validate_unique() except ValidationError as e: if hasattr(model.objects, "get_natural_fields"): fields = model.objects.get_natural_fields() else: fields = e.message_dict.keys() existing_object = model.objects.get(**{k: getattr(obj.object, k) for k in fields}) obj.object.pk = existing_object.pk try: obj.save(using=self.using) if show_progress: self.stdout.write( '\rProcessed %i object(s).' % loaded_objects_in_fixture, ending='' ) # psycopg2 raises ValueError if data contains NUL chars. except (DatabaseError, IntegrityError, ValueError) as e: e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': e, },) raise if objects and show_progress: self.stdout.write('') # add a newline after progress indicator self.loaded_object_count += loaded_objects_in_fixture self.fixture_object_count += objects_in_fixture except Exception as e: if not isinstance(e, CommandError): e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),) raise finally: fixture.close() # Warn if the fixture we loaded contains 0 objects. if objects_in_fixture == 0: warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, RuntimeWarning )
def update_roles_permissions( Role, Permission, RolePermission, app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs ): if not router.allow_migrate_model(using, Role): return try: Role = app_config.get_model("trusts", "Role") except LookupError: return # This will hold the roles we're looking for as # (rolename (permissions)) model_roles = {} for klass in app_config.get_models(): if hasattr(klass._meta, "roles"): _process_roles(Permission, model_roles, klass, klass._meta.roles, using) elif hasattr(klass._meta, "content_roles"): if hasattr(klass, "get_content_model"): content_klass = klass.get_content_model() _process_roles(Permission, model_roles, content_klass, klass._meta.content_roles, using) # Find all the Roles and its Permission db_roles = {} for r in Role.objects.using(using).all(): db_roles[r.name] = set(r.permissions.all()) # Get all the diff between sets model_rolenames = set(model_roles.keys()) db_rolenames = set(db_roles.keys()) added_rolenames = model_rolenames - db_rolenames deleted_rolenames = db_rolenames - model_rolenames existing_rolenames = model_rolenames.intersection(db_rolenames) # Prepare rolepermissions for bulk op at the end bulk_add_rolepermissions = [] q_del_rolepermissions = [] deleted_role_ids = [] # Process added roles for rolename in added_rolenames: r = Role(name=rolename) r.save() for p in model_roles[rolename]: bulk_add_rolepermissions.append(RolePermission(managed=True, permission=p, role=r)) # Process existing roles for rolename in existing_rolenames: r = Role.objects.get(name=rolename) db_permissions = db_roles[rolename] model_permissions = model_roles[rolename] added_permissions = set(model_permissions) - set(db_permissions) for p in added_permissions: bulk_add_rolepermissions.append(RolePermission(managed=True, permission=p, role=r)) deleted_permissions = set(db_permissions) - set(model_permissions) if len(deleted_permissions): q_del_rolepermissions.append((r, Q(managed=True, role=r, permission__in=deleted_permissions))) # Process deleted roles for rolename in deleted_rolenames: r = Role.objects.get(name=rolename) q_del_rolepermissions.append((r, Q(managed=True, role=r))) deleted_role_ids.append(r.pk) # Create the added role permissions RolePermission.objects.using(using).bulk_create(bulk_add_rolepermissions) if verbosity >= 2: for rolepermission in bulk_add_rolepermissions: print('Adding role(%s).rolepermission "%s"' % (rolepermission.role.name, rolepermission)) # Remove the deleted role permissions for r, q in q_del_rolepermissions: qs = RolePermission.objects.filter(q) if verbosity >= 2: if qs.count() > 0: for rolepermission in qs.all(): print('Removing role(%s).rolepermission "%s"' % (rolepermission.role.name, rolepermission)) qs.delete() # Remove the deleted role qs = Role.objects.filter(pk__in=deleted_role_ids, permissions__isnull=True) if verbosity >= 2: if qs.count() > 0: for role in qs.all(): print('Removing role "%s"' % (role.name)) qs.delete()
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): """ Copy/paste from django/contrib/auth/management/__init__.py with small modifications (for_concrete_model, app_config.get_proxies). Create permissions for defined proxy models in apps module. """ from django.apps import apps if not app_config.models_module: return try: Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return from django.contrib.contenttypes.models import ContentType # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() for klass in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model( model=klass, for_concrete_model=False) if klass._meta.proxy: concrete_ctype = ContentType.objects.db_manager( using).get_for_model( # noqa model=klass, ) perms = Permission.objects.using(using).filter( content_type=concrete_ctype, codename__endswith=klass._meta.model_name) if perms: perms.update(content_type=ctype) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta, ctype): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("content_type", "codename")) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] # Validate the permissions before bulk_creation to avoid cryptic # database error when the verbose_name is longer than 50 characters permission_name_max_length = Permission._meta.get_field('name').max_length verbose_name_max_length = permission_name_max_length - len('Can change ') for perm in perms: if len(perm.name) > permission_name_max_length: raise exceptions.ValidationError( "The verbose_name of %s.%s is longer than %s characters" % ( perm.content_type.app_label, perm.content_type.model, verbose_name_max_length, )) Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def update_contenttypes(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): """ Creates content types for models in the given app, removing any model entries that no longer have a matching model class. """ if not app_config.models_module: return app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') except LookupError: return if not router.allow_migrate_model(using, ContentType): return ContentType.objects.clear_cache() # Always clear the global content types cache. if apps is not global_apps: global_apps.get_model('contenttypes', 'ContentType').objects.clear_cache() app_models = { model._meta.model_name: model for model in app_config.get_models()} if not app_models: return # Get all the content types content_types = { ct.model: ct for ct in ContentType.objects.using(using).filter(app_label=app_label) } to_remove = [ ct for (model_name, ct) in six.iteritems(content_types) if model_name not in app_models ] cts = [ ContentType( app_label=app_label, model=model_name, ) for (model_name, model) in six.iteritems(app_models) if model_name not in content_types ] ContentType.objects.using(using).bulk_create(cts) if verbosity >= 2: for ct in cts: print("Adding content type '%s | %s'" % (ct.app_label, ct.model)) # Confirm that the content type is stale before deletion. using = router.db_for_write(ContentType) if to_remove: if interactive: ct_info = [] for ct in to_remove: ct_info.append(' - Content type for %s.%s' % (ct.app_label, ct.model)) collector = NoFastDeleteCollector(using=using) collector.collect([ct]) for obj_type, objs in collector.data.items(): if objs == {ct}: continue ct_info.append(' - %s object%s of type %s.%s:' % ( len(objs), 's' if len(objs) != 1 else '', obj_type._meta.app_label, obj_type._meta.model_name) ) content_type_display = '\n'.join(ct_info) print("""Some content types in your database are stale and can be deleted. Any objects that depend on these content types will then also be deleted. The content types, and the dependent objects that would be deleted, are: %s This list does not include data that might be in your database outside of Django's models. Are you sure you want to delete these content types? If you're unsure, answer 'no'. """ % content_type_display) ok_to_delete = input("Type 'yes' to continue, or 'no' to cancel: ") else: ok_to_delete = False if ok_to_delete == 'yes': for ct in to_remove: if verbosity >= 2: print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model)) ct.delete() else: if verbosity >= 2: print("Stale content types remain.")
def load_label(self, fixture_label): """ Loads fixtures files for a given label. This method is largely copied from django.core.management.commands.loaddata.Command but with the addition of using bulk_create where possible. """ show_progress = self.verbosity >= 3 for fixture_file, fixture_dir, fixture_name in self.find_fixtures( fixture_label): _, ser_fmt, cmp_fmt = self.parse_name( os.path.basename(fixture_file)) open_method, mode = self.compression_formats[cmp_fmt] fixture = open_method(fixture_file, mode) try: self.fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if self.verbosity >= 2: self.stdout.write( "Installing %s fixture '%s' from %s." % (ser_fmt, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize( ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore, ) models_in_file = set() objects_to_create = [] for obj in objects: objects_in_fixture += 1 if (obj.object._meta.app_config in self.excluded_apps or type(obj.object) in self.excluded_models): continue if router.allow_migrate_model(self.using, obj.object.__class__): self.models.add(obj.object.__class__) models_in_file.add(obj.object.__class__) objects_to_create.append(obj) if len(models_in_file) == 1: model = list(models_in_file)[0] try: model_objects = [] m2m_related_objects = defaultdict(list) for obj in objects_to_create: model_objects.append(obj.object) if obj.m2m_data: for accessor_field, m2m_list in obj.m2m_data.items( ): m2m_rel = getattr(model, accessor_field).rel related_model = m2m_rel.model through_model = m2m_rel.through for m2m_id in m2m_list: model_id_field = '{}_id'.format( model._meta.model_name) related_model_id_field = '{}_id'.format( related_model._meta.model_name) m2m_related_objects[ through_model].append( through_model( **{ model_id_field: obj.object.pk, related_model_id_field: m2m_id })) model.objects.bulk_create(model_objects) for m2m_model in m2m_related_objects: m2m_model.objects.bulk_create( m2m_related_objects[m2m_model]) loaded_objects_in_fixture += len(objects_to_create) if show_progress: self.stdout.write('\rProcessed %i object(s).' % loaded_objects_in_fixture, ending='') except (DatabaseError, IntegrityError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s: %(error_msg)s" % { 'app_label': model._meta.app_label, 'object_name': model._meta.object_name, 'error_msg': force_text(e) }, ) raise else: for obj in objects_to_create: try: obj.save(using=self.using) loaded_objects_in_fixture += 1 if show_progress: self.stdout.write('\rProcessed %i object(s).' % loaded_objects_in_fixture, ending='') except (DatabaseError, IntegrityError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) }, ) raise if objects and show_progress: self.stdout.write( '') # add a newline after progress indicator self.loaded_object_count += loaded_objects_in_fixture self.fixture_object_count += objects_in_fixture except Exception as e: if not isinstance(e, CommandError): e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e), ) raise finally: fixture.close() # Warn if the fixture we loaded contains 0 objects. if objects_in_fixture == 0: warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, RuntimeWarning)
def create_application_dynamic_permissions(self, app_config, db_alias=DEFAULT_DB_ALIAS, **kwargs): from collections import OrderedDict from django.apps import apps from django.core.exceptions import ImproperlyConfigured from django.db import IntegrityError from django.db import router try: ContentType = apps.get_model('contenttypes', 'ContentType') ModelAction = apps.get_model('talos', 'ModelAction') ObjectAction = apps.get_model('talos', 'ObjectAction') except LookupError: return if not router.allow_migrate_model(db_alias, ModelAction): return if not router.allow_migrate_model(db_alias, ObjectAction): return model_action_dict = OrderedDict() object_action_dict = OrderedDict() for model in app_config.get_models(): if hasattr(model._meta, '_talos_model_info'): model_info = model._meta._talos_model_info if model_info.has_actions(): content_type = ContentType.objects.db_manager( db_alias).get_for_model(model_info.model) for action_info in model_info.model_action_infos.values(): model_action_dict[action_info.code] = ModelAction( application=model._meta.app_config.label, model=model._meta.label_lower, action=action_info.action, code=action_info.code, name=action_info.name, content_type=content_type) for action_info in model_info.object_action_infos.values(): object_action_dict[action_info.code] = ObjectAction( application=model._meta.app_config.label, model=model._meta.label_lower, action=action_info.action, code=action_info.code, name=action_info.name, content_type=content_type) existing_model_action_codes = list( ModelAction.objects.using(db_alias).filter( code__in=model_action_dict.keys()).values_list('code', flat=True)) existing_object_action_codes = list( ObjectAction.objects.using(db_alias).filter( code__in=object_action_dict.keys()).values_list('code', flat=True)) missing_model_action_dict = { code: model_action_dict[code] for code in model_action_dict if code not in existing_model_action_codes } missing_object_action_dict = { code: object_action_dict[code] for code in object_action_dict if code not in existing_object_action_codes } try: ModelAction.objects.using(db_alias).bulk_create( missing_model_action_dict.values()) except IntegrityError: for model_action in missing_model_action_dict.values(): try: model_action.save() except IntegrityError: raise ImproperlyConfigured( INVALID_PERMISSION_NAME_MESSAGE.format( model_action.name)) try: ObjectAction.objects.using(db_alias).bulk_create( missing_object_action_dict.values()) except IntegrityError: for object_action in missing_object_action_dict.values(): try: object_action.save() except IntegrityError: raise ImproperlyConfigured( INVALID_PERMISSION_NAME_MESSAGE.format( object_action.name))
def update_contenttypes(sender, verbosity=2, db=DEFAULT_DB_ALIAS, **kwargs): """ Django's default update_contenttypes relies on many inconsistent queries which causes problems with syncdb. This monkeypatch replaces it with a version that does look ups on unique constraints which are slightly better protected from eventual consistency issues by the context cache. """ from django.contrib.contenttypes.models import ContentType if verbosity >= 2: print("Running Djangae version of update_contenttypes on {}".format(sender)) try: apps.get_model('contenttypes', 'ContentType') except LookupError: return if hasattr(router, "allow_migrate_model"): if not router.allow_migrate_model(db, ContentType): return ContentType.objects.clear_cache() app_models = sender.get_models() if not app_models: return # They all have the same app_label, get the first one. app_label = sender.label app_models = dict( (model._meta.model_name, model) for model in app_models ) created_or_existing_pks = [] created_or_existing_by_unique = {} for (model_name, model) in app_models.items(): # Go through get_or_create any models that we want to keep defaults = {} if django.VERSION < (1, 9): defaults['name'] = smart_text(model._meta.verbose_name_raw) ct, created = ContentType.objects.get_or_create( app_label=app_label, model=model_name, defaults=defaults, ) if verbosity >= 2 and created: print("Adding content type '%s | %s'" % (ct.app_label, ct.model)) created_or_existing_pks.append(ct.pk) created_or_existing_by_unique[(app_label, model_name)] = ct.pk # Now lets see if we should remove any to_remove = [x for x in ContentType.objects.filter(app_label=app_label) if x.pk not in created_or_existing_pks] # Now it's possible that our get_or_create failed because of consistency issues and we create a duplicate. # Then the original appears in the to_remove and we remove the original. This is bad. So here we go through the # to_remove list, and if we created the content type just now, we delete that one, and restore the original in the # cache for ct in to_remove: unique = (ct.app_label, ct.model) if unique in created_or_existing_by_unique: # We accidentally created a duplicate above due to HRD issues, delete the one we created ContentType.objects.get(pk=created_or_existing_by_unique[unique]).delete() created_or_existing_by_unique[unique] = ct.pk ct.save() # Recache this one in the context cache to_remove = [ x for x in to_remove if (x.app_label, x.model) not in created_or_existing_by_unique ] # Now, anything left should actually be a stale thing. It's still possible we missed some but they'll get picked up # next time. Confirm that the content type is stale before deletion. if to_remove: if kwargs.get('interactive', False): content_type_display = '\n'.join([ ' %s | %s' % (x.app_label, x.model) for x in to_remove ]) ok_to_delete = input("""The following content types are stale and need to be deleted: %s Any objects related to these content types by a foreign key will also be deleted. Are you sure you want to delete these content types? If you're unsure, answer 'no'. Type 'yes' to continue, or 'no' to cancel: """ % content_type_display) else: ok_to_delete = False if ok_to_delete == 'yes': for ct in to_remove: if verbosity >= 2: print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model)) ct.delete() else: if verbosity >= 2: print("Stale content types remain.")
def allow_migrate(model): for db in connections: if router.allow_migrate_model(db, model): yield db
def create_report_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): """ Copied from django create permissions for model, edited to create view and print for reports Reports permissions are created in the database similar to django's app.print_clientbalances (ClientBalances is the report slug (defaulting to report class name), Content Type is the base model content_type :param app_config: :param verbosity: :param interactive: :param using: :param apps: :param kwargs: :return: """ if not app_config.models_module: return # Ensure that contenttypes are created for this app. Needed if # 'django.contrib.auth' is in INSTALLED_APPS before # 'django.contrib.contenttypes'. create_contenttypes(app_config, verbosity=verbosity, interactive=interactive, using=using, apps=apps, **kwargs) app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return # This will hold the permissions we're looking for as # (content_type, (codename, name)) all_reports = report_registry.get_all_reports() searched_perms = [] # The codenames and ctypes that should exist. ctypes = set() for report in all_reports: # for klass in report_registry.get_base_models(): # Force looking up the content types in the current database # before creating foreign keys to them. klass = report.base_model ctype = ContentType.objects.db_manager(using).get_for_model( klass, for_concrete_model=False) ctypes.add(ctype) for perm in _get_all_permissions(report): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("content_type", "codename")) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def update_contenttypes(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): """ Creates content types for models in the given app, removing any model entries that no longer have a matching model class. """ if not app_config.models_module: return app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') except LookupError: return if not router.allow_migrate_model(using, ContentType): return ContentType.objects.clear_cache() # Always clear the global content types cache. if apps is not global_apps: global_apps.get_model('contenttypes', 'ContentType').objects.clear_cache() app_models = { model._meta.model_name: model for model in app_config.get_models() } if not app_models: return # Get all the content types content_types = { ct.model: ct for ct in ContentType.objects.using(using).filter(app_label=app_label) } to_remove = [ ct for (model_name, ct) in six.iteritems(content_types) if model_name not in app_models ] cts = [ ContentType( app_label=app_label, model=model_name, ) for (model_name, model) in six.iteritems(app_models) if model_name not in content_types ] ContentType.objects.using(using).bulk_create(cts) if verbosity >= 2: for ct in cts: print("Adding content type '%s | %s'" % (ct.app_label, ct.model)) # Confirm that the content type is stale before deletion. using = router.db_for_write(ContentType) if to_remove: if interactive: ct_info = [] for ct in to_remove: ct_info.append(' - Content type for %s.%s' % (ct.app_label, ct.model)) collector = NoFastDeleteCollector(using=using) collector.collect([ct]) for obj_type, objs in collector.data.items(): if objs == {ct}: continue ct_info.append(' - %s %s object(s)' % ( len(objs), obj_type._meta.label, )) content_type_display = '\n'.join(ct_info) print( """Some content types in your database are stale and can be deleted. Any objects that depend on these content types will also be deleted. The content types and dependent objects that would be deleted are: %s This list doesn't include any cascade deletions to data outside of Django's models (uncommon). Are you sure you want to delete these content types? If you're unsure, answer 'no'. """ % content_type_display) ok_to_delete = input("Type 'yes' to continue, or 'no' to cancel: ") else: ok_to_delete = False if ok_to_delete == 'yes': for ct in to_remove: if verbosity >= 2: print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model)) ct.delete() else: if verbosity >= 2: print("Stale content types remain.")
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): if not app_config.models_module: return app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = [] # The codenames and ctypes that should exist. ctypes = set() for klass in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. # For djongo, replace this # all_perms = set(Permission.objects.using(using).filter( # content_type__in=ctypes, # ).values_list( # "content_type", "codename" # )) # With this all_perms = set() ctypes_pks = set(ct.pk for ct in ctypes) for ctype, codename in Permission.objects.all().values_list( 'content_type', 'codename')[:1000000]: if ctype in ctypes_pks: all_perms.add((ctype, codename)) # End of the djongo edit perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def create_view_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): # noqa """ Create 'view' permissions for all models. ``django.contrib.auth`` only creates add, change and delete permissions. Since we want to support read-only views, we need to add our own permission. Copied from ``https://github.com/django/django/blob/1.9.6/django/contrib/auth/management/__init__.py#L60``. """ if not app_config.models_module: return try: Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return from django.contrib.contenttypes.models import ContentType # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() for klass in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) perm = (get_permission_codename('view', klass._meta), 'Can view %s' % (klass._meta.verbose_name_raw)) searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("content_type", "codename")) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] # Validate the permissions before bulk_creation to avoid cryptic # database error when the verbose_name is longer than 50 characters permission_name_max_length = Permission._meta.get_field('name').max_length verbose_name_max_length = permission_name_max_length - 11 # len('Can change ') prefix for perm in perms: if len(perm.name) > permission_name_max_length: raise ValidationError( "The verbose_name of %s.%s is longer than %s characters" % ( perm.content_type.app_label, perm.content_type.model, verbose_name_max_length, )) Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def create_extra_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): """ Create the extra user & group permissions we need. Pulled almost entirely from: django.contrib.auth.management:create_permissions """ if not app_config.models_module: return try: Permission = apps.get_model('auth', 'Permission') User = apps.get_model('auth', 'User') Group = apps.get_model('auth', 'Group') except LookupError: return if not router.allow_migrate_model(using, Permission): return from django.contrib.contenttypes.models import ContentType # The list of perms we need to create perms = [] for model, extra_perms in ((User, EXTRA_USER_PERMS), (Group, EXTRA_GROUP_PERMS)): # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = [] # Grab the ctype for the model ctype = ContentType.objects.db_manager(using).get_for_model(model) model_name = model._meta.model_name for perm in extra_perms: searched_perms.append((ctype, ('%s_%s' % (perm, model_name), 'Can %s %s' % (perm, model_name)))) # Find all the Permissions that have a content_type for the model. # We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set(Permission.objects.using(using).filter( content_type=ctype, ).values_list( "content_type", "codename" )) perms += [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] # Validate the permissions before bulk_creation to avoid cryptic # database error when the verbose_name is longer than 50 characters permission_name_max_length = Permission._meta.get_field('name').max_length verbose_name_max_length = permission_name_max_length - 11 # len('Can change ') prefix for perm in perms: if len(perm.name) > permission_name_max_length: raise exceptions.ValidationError( "The verbose_name of %s.%s is longer than %s characters" % ( perm.content_type.app_label, perm.content_type.model, verbose_name_max_length, ) ) Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
from collections import OrderedDict
def get_contenttypes_and_models(app_config, using, ContentType): if not router.allow_migrate_model(using, ContentType): return None, None
def handle(self, **options): db = options['database'] include_stale_apps = options['include_stale_apps'] interactive = options['interactive'] verbosity = options['verbosity'] if not router.allow_migrate_model(db, ContentType): return ContentType.objects.clear_cache() apps_content_types = itertools.groupby( ContentType.objects.using(db).order_by('app_label', 'model'), lambda obj: obj.app_label, ) for app_label, content_types in apps_content_types: if not include_stale_apps and app_label not in apps.app_configs: continue to_remove = [ ct for ct in content_types if ct.model_class() is None ] # Confirm that the content type is stale before deletion. using = router.db_for_write(ContentType) if to_remove: if interactive: ct_info = [] for ct in to_remove: ct_info.append(' - Content type for %s.%s' % (ct.app_label, ct.model)) collector = NoFastDeleteCollector(using=using) collector.collect([ct]) for obj_type, objs in collector.data.items(): if objs != {ct}: ct_info.append(' - %s %s object(s)' % ( len(objs), obj_type._meta.label, )) content_type_display = '\n'.join(ct_info) self.stdout.write( """Some content types in your database are stale and can be deleted. Any objects that depend on these content types will also be deleted. The content types and dependent objects that would be deleted are: %s This list doesn't include any cascade deletions to data outside of Django's models (uncommon). Are you sure you want to delete these content types? If you're unsure, answer 'no'.""" % content_type_display) ok_to_delete = input( "Type 'yes' to continue, or 'no' to cancel: ") else: ok_to_delete = 'yes' if ok_to_delete == 'yes': for ct in to_remove: if verbosity >= 2: self.stdout.write( "Deleting stale content type '%s | %s'" % (ct.app_label, ct.model)) ct.delete() else: if verbosity >= 2: self.stdout.write("Stale content types remain.")
def update_contenttypes(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): """ Creates content types for models in the given app, removing any model entries that no longer have a matching model class. """ if not app_config.models_module: return try: ContentType = apps.get_model('contenttypes', 'ContentType') except LookupError: return if not router.allow_migrate_model(using, ContentType): return ContentType.objects.clear_cache() app_label = app_config.label app_models = { model._meta.model_name: model for model in app_config.get_models()} if not app_models: return # Get all the content types content_types = { ct.model: ct for ct in ContentType.objects.using(using).filter(app_label=app_label) } to_remove = [ ct for (model_name, ct) in six.iteritems(content_types) if model_name not in app_models ] cts = [ ContentType( app_label=app_label, model=model_name, ) for (model_name, model) in six.iteritems(app_models) if model_name not in content_types ] ContentType.objects.using(using).bulk_create(cts) if verbosity >= 2: for ct in cts: print("Adding content type '%s | %s'" % (ct.app_label, ct.model)) # Confirm that the content type is stale before deletion. if to_remove: if interactive: content_type_display = '\n'.join( ' %s | %s' % (ct.app_label, ct.model) for ct in to_remove ) ok_to_delete = input("""The following content types are stale and need to be deleted: %s Any objects related to these content types by a foreign key will also be deleted. Are you sure you want to delete these content types? If you're unsure, answer 'no'. Type 'yes' to continue, or 'no' to cancel: """ % content_type_display) else: ok_to_delete = False if ok_to_delete == 'yes': for ct in to_remove: if verbosity >= 2: print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model)) ct.delete() else: if verbosity >= 2: print("Stale content types remain.")
def get_objects(count_only=False): """ Collate the objects to be serialized. If count_only is True, just count the number of objects to be serialized. """ models = serializers.sort_dependencies(app_list.items()) for model in models: if model in excluded_models: continue if model._meta.proxy and model._meta.proxy_for_model not in models: warnings.warn( "%s is a proxy model and won't be serialized." % model._meta.label, category=ProxyModelWarning, ) if not model._meta.proxy and router.allow_migrate_model(using, model): if use_base_manager: objects = model._base_manager else: objects = model._default_manager queryset = objects.using(using).order_by(model._meta.pk.name) if primary_keys: queryset = queryset.filter(pk__in=primary_keys) if count_only: yield queryset.order_by().count() else: <<<<<<< HEAD for obj in queryset.iterator(): yield obj =======
def create_proxy_permissions( app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): if not app_config.models_module: return logger = logging.getLogger(__name__) # print(app_config) try: logger.info("Tentando obter modelo de permissão do app.") Permission = django.apps.apps.get_model('auth', 'Permission') except LookupError as e: logger.error(str(e)) return if not router.allow_migrate_model(using, Permission): return from django.contrib.contenttypes.models import ContentType permission_name_max_length = Permission._meta.get_field('name').max_length # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() for klass in list(app_config.get_models()): opts = klass._meta permissions = ( ("list_" + opts.model_name, string_concat( _('Visualizaçao da lista de'), ' ', opts.verbose_name_plural)), ("detail_" + opts.model_name, string_concat( _('Visualização dos detalhes de'), ' ', opts.verbose_name_plural)), ) opts.permissions = tuple( set(list(permissions) + list(opts.permissions))) if opts.proxy: # Force looking up the content types in the current database # before creating foreign keys to them. app_label, model = opts.app_label, opts.model_name try: logger.info("Tentando obter db_manager.") ctype = ContentType.objects.db_manager( using).get_by_natural_key(app_label, model) except Exception as e: logger.error(str(e)) ctype = ContentType.objects.db_manager( using).create(app_label=app_label, model=model) else: ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) # FIXME: Retirar try except quando sapl passar a usar django 1.11 try: logger.info("_get_all_permissions") # Função não existe mais em Django 1.11 # como sapl ainda não foi para Django 1.11 # esta excessão foi adicionada para caso o # Sapl esteja rodando em um projeto 1.11 não ocorra erros _all_perms_of_klass = _get_all_permissions(klass._meta, ctype) except Exception as e: logger.error(str(e)) # Nova função usada em projetos com Django 1.11 e o sapl é uma app _all_perms_of_klass = _get_all_permissions(klass._meta) for perm in _all_perms_of_klass: searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set(Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list( "content_type", "codename" )) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] # Validate the permissions before bulk_creation to avoid cryptic database # error when the name is longer than 255 characters for perm in perms: if len(perm.name) > permission_name_max_length: logger.error("The permission name %s of %s.%s " "is longer than %s characters" % ( perm.name, perm.content_type.app_label, perm.content_type.model, permission_name_max_length, )) raise exceptions.ValidationError( 'The permission name %s of %s.%s ' 'is longer than %s characters' % ( perm.name, perm.content_type.app_label, perm.content_type.model, permission_name_max_length, ) ) Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def handle(self, *app_labels, **options): format = options.get('format') indent = options.get('indent') using = options.get('database') excludes = options.get('exclude') output = options.get('output') output_folder = options.get('output_folder') print "Output folder:", output_folder print "NOTE: See --output-folder option" max_records_per_chunk = options.get('max_records_per_chunk') show_traceback = options.get('traceback') use_natural_foreign_keys = options.get('use_natural_foreign_keys') use_natural_primary_keys = options.get('use_natural_primary_keys') use_base_manager = options.get('use_base_manager') pks = options.get('primary_keys') if pks: primary_keys = pks.split(',') else: primary_keys = [] excluded_apps = set() excluded_models = set() for exclude in excludes: if '.' in exclude: try: model = apps.get_model(exclude) except LookupError: raise CommandError('Unknown model in excludes: %s' % exclude) excluded_models.add(model) else: try: app_config = apps.get_app_config(exclude) except LookupError as e: raise CommandError(str(e)) excluded_apps.add(app_config) if len(app_labels) == 0: if primary_keys: raise CommandError( "You can only use --pks option with one model") app_list = OrderedDict((app_config, None) for app_config in apps.get_app_configs() if app_config.models_module is not None and app_config not in excluded_apps) else: if len(app_labels) > 1 and primary_keys: raise CommandError( "You can only use --pks option with one model") app_list = OrderedDict() for label in app_labels: try: app_label, model_label = label.split('.') try: app_config = apps.get_app_config(app_label) except LookupError as e: raise CommandError(str(e)) if app_config.models_module is None or app_config in excluded_apps: continue try: model = app_config.get_model(model_label) except LookupError: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) app_list_value = app_list.setdefault(app_config, []) # We may have previously seen a "all-models" request for # this app (no model qualifier was given). In this case # there is no need adding specific models to the list. if app_list_value is not None: if model not in app_list_value: app_list_value.append(model) except ValueError: if primary_keys: raise CommandError( "You can only use --pks option with one model") # This is just an app - no model qualifier app_label = label try: app_config = apps.get_app_config(app_label) except LookupError as e: raise CommandError(str(e)) if app_config.models_module is None or app_config in excluded_apps: continue app_list[app_config] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): try: serializers.get_serializer(format) except serializers.SerializerDoesNotExist: pass raise CommandError("Unknown serialization format: %s" % format) # def get_objects(count_only=False): # """ # Collate the objects to be serialized. If count_only is True, just # count the number of objects to be serialized. # """ # for model in serializers.sort_dependencies(app_list.items()): # if model in excluded_models: # continue # if not model._meta.proxy and router.allow_migrate_model(using, model): # if use_base_manager: # objects = model._base_manager # else: # objects = model._default_manager # # queryset = objects.using(using).order_by(model._meta.pk.name) # if primary_keys: # queryset = queryset.filter(pk__in=primary_keys) # if count_only: # yield queryset.order_by().count() # else: # for obj in queryset.iterator(): # yield obj # # try: # self.stdout.ending = None # progress_output = None # object_count = 0 # # If dumpdata is outputting to stdout, there is no way to display progress # if (output and self.stdout.isatty() and options['verbosity'] > 0): # progress_output = self.stdout # object_count = sum(get_objects(count_only=True)) # stream = open(output, 'w') if output else None # try: # chunk_count = 1000 # serializers.serialize(format, get_objects(), indent=indent, # use_natural_foreign_keys=use_natural_foreign_keys, # use_natural_primary_keys=use_natural_primary_keys, # stream=stream or self.stdout, progress_output=progress_output, # object_count=object_count) # finally: # if stream: # stream.close() # except Exception as e: # if show_traceback: # raise # raise CommandError("Unable to serialize database: %s" % e) objects = [] model_count = 1000 chunk_count = 1000 for model in serializers.sort_dependencies(app_list.items()): model_count += 1 if model in excluded_models: continue if not model._meta.proxy and router.allow_migrate_model( using, model): if use_base_manager: objects.extend(model._base_manager.using(using).all()) else: items_total = model._default_manager.using(using).count() chunks_total = (items_total / max_records_per_chunk) + 1 for chunk_num in range(0, chunks_total): output_objects = model._default_manager.using( using).all().order_by( model._meta.pk.name)[chunk_num * max_records_per_chunk: (chunk_num + 1) * max_records_per_chunk] if output_objects: chunk_count += 1 dump_file_name = output_folder + "/%d_%d.json" % ( model_count, chunk_count) print "Dumping file: %s [%d]" % (dump_file_name, chunks_total) output = serializers.serialize( format, output_objects, indent=indent, use_natural_foreign_keys= use_natural_foreign_keys, use_natural_primary_keys= use_natural_primary_keys) with open(dump_file_name, "w") as dumpfile: dumpfile.write(output)
def update_contenttypes(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): """ Creates content types for models in the given app, removing any model entries that no longer have a matching model class. """ if not app_config.models_module: return try: ContentType = apps.get_model('contenttypes', 'ContentType') except LookupError: return if not router.allow_migrate_model(using, ContentType): return ContentType.objects.clear_cache() app_label = app_config.label app_models = { model._meta.model_name: model for model in app_config.get_models() } if not app_models: return # Get all the content types content_types = { ct.model: ct for ct in ContentType.objects.using(using).filter(app_label=app_label) } to_remove = [ ct for (model_name, ct) in six.iteritems(content_types) if model_name not in app_models ] cts = [ ContentType( app_label=app_label, model=model_name, ) for (model_name, model) in six.iteritems(app_models) if model_name not in content_types ] ContentType.objects.using(using).bulk_create(cts) if verbosity >= 2: for ct in cts: print("Adding content type '%s | %s'" % (ct.app_label, ct.model)) # Confirm that the content type is stale before deletion. if to_remove: if interactive: content_type_display = '\n'.join( ' %s | %s' % (ct.app_label, ct.model) for ct in to_remove) ok_to_delete = input( """The following content types are stale and need to be deleted: %s Any objects related to these content types by a foreign key will also be deleted. Are you sure you want to delete these content types? If you're unsure, answer 'no'. Type 'yes' to continue, or 'no' to cancel: """ % content_type_display) else: ok_to_delete = False if ok_to_delete == 'yes': for ct in to_remove: if verbosity >= 2: print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model)) ct.delete() else: if verbosity >= 2: print("Stale content types remain.")
def create_proxy_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): if not app_config.models_module: return # print(app_config) try: Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return from django.contrib.contenttypes.models import ContentType permission_name_max_length = Permission._meta.get_field('name').max_length # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() for klass in list(app_config.get_models()): opts = klass._meta permissions = ( ("list_" + opts.model_name, string_concat(_('Visualizaçao da lista de'), ' ', opts.verbose_name_plural)), ("detail_" + opts.model_name, string_concat(_('Visualização dos detalhes de'), ' ', opts.verbose_name_plural)), ) opts.permissions = tuple( set(list(permissions) + list(opts.permissions))) if opts.proxy: # Force looking up the content types in the current database # before creating foreign keys to them. app_label, model = opts.app_label, opts.model_name try: ctype = ContentType.objects.db_manager( using).get_by_natural_key(app_label, model) except: ctype = ContentType.objects.db_manager(using).create( app_label=app_label, model=model) else: ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta, ctype): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("content_type", "codename")) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] # Validate the permissions before bulk_creation to avoid cryptic database # error when the name is longer than 255 characters for perm in perms: if len(perm.name) > permission_name_max_length: raise exceptions.ValidationError('The permission name %s of %s.%s ' 'is longer than %s characters' % ( perm.name, perm.content_type.app_label, perm.content_type.model, permission_name_max_length, )) Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def create_table(self, database, tablename, dry_run): cache = BaseDatabaseCache(tablename, {}) if not router.allow_migrate_model(database, cache.cache_model_class): return connection = connections[database] if tablename in connection.introspection.table_names(): if self.verbosity > 0: self.stdout.write("Cache table '%s' already exists." % tablename) return fields = ( # "key" is a reserved word in MySQL, so use "cache_key" instead. models.CharField( name="cache_key", max_length=255, unique=True, primary_key=True ), models.TextField(name="value"), models.DateTimeField(name="expires", db_index=True), ) table_output = [] index_output = [] qn = connection.ops.quote_name for f in fields: field_output = [ qn(f.name), f.db_type(connection=connection), "%sNULL" % ("NOT " if not f.null else ""), ] if f.primary_key: field_output.append("PRIMARY KEY") elif f.unique: field_output.append("UNIQUE") if f.db_index: unique = "UNIQUE " if f.unique else "" index_output.append( "CREATE %sINDEX %s ON %s (%s);" % ( unique, qn("%s_%s" % (tablename, f.name)), qn(tablename), qn(f.name), ) ) table_output.append(" ".join(field_output)) full_statement = ["CREATE TABLE %s (" % qn(tablename)] for i, line in enumerate(table_output): full_statement.append( " %s%s" % (line, "," if i < len(table_output) - 1 else "") ) full_statement.append(");") full_statement = "\n".join(full_statement) if dry_run: self.stdout.write(full_statement) for statement in index_output: self.stdout.write(statement) return with transaction.atomic( using=database, savepoint=connection.features.can_rollback_ddl ): with connection.cursor() as curs: try: curs.execute(full_statement) except DatabaseError as e: raise CommandError( "Cache table '%s' could not be created.\nThe error was: %s." % (tablename, e) ) for statement in index_output: curs.execute(statement) if self.verbosity > 1: self.stdout.write("Cache table '%s' created." % tablename)
def create_permissions( # NOQA app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs ): if not app_config.models_module: return from field_permissions.registry import field_permissions app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model("contenttypes", "ContentType") # NOQA Permission = apps.get_model("auth", "Permission") # NOQA except LookupError: return if not router.allow_migrate_model(using, Permission): return # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = [] # The codenames and ctypes that should exist. ctypes = set() for klass in app_config.get_models(): if not field_permissions.in_registry(klass): continue # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) for perm in field_permissions.get_field_permissions_for_model(klass): searched_perms.append((ctype, perm)) if not searched_perms: return # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using) .filter(content_type__in=ctypes) .values_list("content_type", "codename") ) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '{}'".format(perm.codename)) # NOQA
def load_label(self, fixture_label): """ Loads fixtures files for a given label. """ show_progress = self.verbosity >= 3 for fixture_file, fixture_dir, fixture_name in self.find_fixtures( fixture_label): _, ser_fmt, cmp_fmt = self.parse_name( os.path.basename(fixture_file)) open_method, mode = self.compression_formats[cmp_fmt] fixture = open_method(fixture_file, mode) try: self.fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if self.verbosity >= 2: self.stdout.write( "Installing %s fixture '%s' from %s." % (ser_fmt, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize( ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore) for obj in objects: objects_in_fixture += 1 if router.allow_migrate_model(self.using, obj.object.__class__): loaded_objects_in_fixture += 1 self.models.add(obj.object.__class__) try: obj.save(using=self.using) if show_progress: self.stdout.write('\rProcessed %i object(s).' % loaded_objects_in_fixture, ending='') except (DatabaseError, IntegrityError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) }, ) raise if objects and show_progress: self.stdout.write( '') # add a newline after progress indicator self.loaded_object_count += loaded_objects_in_fixture self.fixture_object_count += objects_in_fixture except Exception as e: if not isinstance(e, CommandError): e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e), ) raise finally: fixture.close() # Warn if the fixture we loaded contains 0 objects. if objects_in_fixture == 0: warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, RuntimeWarning)
import sys
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): if not app_config.models_module: return # Ensure that contenttypes are created for this BLOG. Needed if # 'django.contrib.auth' is in INSTALLED_APPS before # 'django.contrib.contenttypes'. create_contenttypes(app_config, verbosity=verbosity, interactive=interactive, using=using, apps=apps, **kwargs) app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = [] # The codenames and ctypes that should exist. ctypes = set() for klass in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model( klass, for_concrete_model=False) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("content_type", "codename")) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def create(self, request): objects_in_fixture = 0 loaded_objects_count = 0 models = set() fixture = request.data.get('fixture') if fixture: with transaction.atomic(): connection = connections[DEFAULT_DB_ALIAS] with connection.constraint_checks_disabled(): objects = Deserializer(fixture, ignorenonexistent=True, handle_forward_references=True) objs_with_deferred_fields = [] for obj in objects: objects_in_fixture += 1 if router.allow_migrate_model(DEFAULT_DB_ALIAS, obj.object.__class__): loaded_objects_count += 1 models.add(obj.object.__class__) try: obj.save() # psycopg2 raises ValueError if data contains NUL chars. except (DatabaseError, IntegrityError, ValueError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': e, }, ) raise if obj.deferred_fields: objs_with_deferred_fields.append(obj) for obj in objs_with_deferred_fields: obj.save_deferred_fields() if loaded_objects_count > 0: sequence_sql = connection.ops.sequence_reset_sql( no_style(), models) if sequence_sql: with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) # Close the DB connection -- unless we're still in a transaction. This # is required as a workaround for an edge case in MySQL: if the same # connection is used to create tables, load data, and query, the query # can return incorrect results. See Django #7572, MySQL #37735. if transaction.get_autocommit(): connections[DEFAULT_DB_ALIAS].close() return Response({"objects_loaded": loaded_objects_count}, status=status.HTTP_200_OK)
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): # run auth's create_permissions since we are re-writing the signal handler auth_perms(app_config, verbosity, interactive, using, apps, **kwargs) if not app_config.models_module: return app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') Group = apps.get_model('auth', 'Group') except LookupError: return if not router.allow_migrate_model(using, Permission): return # The ctypes we will review ctypes = set() for model in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(model) ctypes.add(ctype) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("codename", flat=True)) def post_process_urlpatterns(urlpatterns, base='', namespace=None): patterns, resolvers = {}, {} for p in urlpatterns: if isinstance(p, RegexURLPattern): try: if not p.name: name = p.name elif namespace: name = '{0}:{1}'.format(namespace, p.name) else: name = p.name if name in patterns: raise ValueError('"{}" is duplicate') # we are presuming only CBV... could handle FBV one day view_class = p.callback.view_class if hasattr( p.callback, 'view_class') else None view_model = None if view_class: view_initkwargs = p.callback.view_initkwargs view_controller = p.callback.view_initkwargs.get( 'controller') view_model = view_controller.model if view_controller else None if view_model: content_type = ContentType.objects.get_for_model( view_model) if content_type not in ctypes: continue view_mode = view_class.mode or view_initkwargs['mode'] verbose_name_plural = view_model._meta.verbose_name_plural else: # we cannot add perms on models not in the calling model's app # we will not add perms for non-model views because Perms do not allow # we will always create and refresh groups continue content_type = None name_parts = name.split(':') view_mode = name_parts[-1] verbose_name_plural = '{}'.format(':'.join( name_parts[:-1])) permission_name = 'Can {} {}'.format( view_mode, verbose_name_plural) permission = Permission(codename=name, name=permission_name, content_type=content_type) patterns[name] = (base + p.regex.pattern, p.callback, permission) except ViewDoesNotExist: continue elif isinstance(p, RegexURLResolver): try: url_patterns = p.url_patterns except ImportError: continue if namespace and p.namespace: _namespace = '{0}:{1}'.format(namespace, p.namespace) else: _namespace = (p.namespace or namespace) if isinstance(p, LocaleRegexURLResolver): for langauge in settings.LANGUAGES: with translation.override(langauge[0]): if _namespace in resolvers: raise ValueError('"{}" is duplicate') resolvers[_namespace] = post_process_urlpatterns( url_patterns, base + p.regex.pattern, namespace=_namespace) else: if _namespace in resolvers: raise ValueError('"{}" is duplicate') resolvers[_namespace] = post_process_urlpatterns( url_patterns, base + p.regex.pattern, namespace=_namespace) # bulk create the permissions all_patterns = resolvers[_namespace]['patterns'] new_patterns, new_permissions = [], [] for name, data in all_patterns.items(): if name not in all_perms: new_patterns.append(name) new_permissions.append(data[2]) print("Adding permission '%s'" % name) Permission.objects.using(using).bulk_create(new_permissions) # get-or-create and update the group group, created = Group.objects.using(using).get_or_create( name=_namespace) if created: print("Adding group '%s'" % _namespace) missing_patterns = new_patterns else: missing_patterns = set(new_patterns) - set( group.permissions.values_list('codename', flat=True)) if missing_patterns: new_group_permissions = Permission.objects.filter( codename__in=missing_patterns) group.permissions.add(*new_group_permissions) for permission in new_group_permissions: print("Adding permission '%s' to group '%s'" % (permission.codename, _namespace)) """ Can't do nested groups yet so we will rely on controller traversal. group_groups = Group.objects.filter(name__in=resolvers) group.groups.add(*group_groups) """ return dict(patterns=patterns, resolvers=resolvers) post_process_urlpatterns(get_backend().urls)
def load_label(self, fixture_label): """Load fixtures files for a given label.""" show_progress = self.verbosity >= 3 for fixture_file, fixture_dir, fixture_name in self.find_fixtures( fixture_label): _, ser_fmt, cmp_fmt = self.parse_name( os.path.basename(fixture_file)) open_method, mode = self.compression_formats[cmp_fmt] fixture = open_method(fixture_file, mode) try: self.fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if self.verbosity >= 2: self.stdout.write( "Installing %s fixture '%s' from %s." % (ser_fmt, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize( ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore, handle_forward_references=True, ) for obj in objects: objects_in_fixture += 1 if (obj.object._meta.app_config in self.excluded_apps or type(obj.object) in self.excluded_models): continue if router.allow_migrate_model(self.using, obj.object.__class__): loaded_objects_in_fixture += 1 self.models.add(obj.object.__class__) try: obj.save(using=self.using) if show_progress: self.stdout.write( "\rProcessed %i object(s)." % loaded_objects_in_fixture, ending="", ) # psycopg2 raises ValueError if data contains NUL chars. except (DatabaseError, IntegrityError, ValueError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { "app_label": obj.object._meta.app_label, "object_name": obj.object._meta.object_name, "pk": obj.object.pk, "error_msg": e, }, ) raise if obj.deferred_fields: self.objs_with_deferred_fields.append(obj) if objects and show_progress: self.stdout.write( "") # add a newline after progress indicator self.loaded_object_count += loaded_objects_in_fixture self.fixture_object_count += objects_in_fixture except Exception as e: if not isinstance(e, CommandError): e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e), ) raise finally: fixture.close() # Warn if the fixture we loaded contains 0 objects. if objects_in_fixture == 0: warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, RuntimeWarning, )
def create_default_environments(app_config, verbosity=2, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): try: Environment = apps.get_model('configfactory', 'Environment') except LookupError: return if not router.allow_migrate_model(using, Environment): return environments = settings.DEFAULT_ENVIRONMENTS aliases = set() # Create or active environments for order, environment_data in enumerate(environments, start=1): alias = environment_data['alias'] # type: str name = environment_data.get('name', alias.title()) aliases.add(alias) try: environment = Environment.objects.using(using).get(alias=alias) if verbosity >= 2: print(f"Updating {alias} environment.") environment.name = name environment.order = order environment.is_active = True environment.save(using=using) except Environment.DoesNotExist: environment = Environment(alias=alias, name=name, order=order) if verbosity >= 2: print(f"Creating {alias} environment.") environment.save(using=using) # Deactivate old environment for environment in Environment.objects.using(using).all(): alias = environment.alias aliases.add(alias) if alias not in aliases and alias != settings.BASE_ENVIRONMENT: if verbosity >= 2: print(f"Deactivating {alias} environment.") environment.is_active = False environment.save(using=using) # Set environment fallback for environment_data in environments: alias = environment_data['alias'] fallback = environment_data.get('fallback') try: environment = Environment.objects.using(using).get(alias=alias) fallback_environment = Environment.objects.using(using).filter( alias=fallback).first() if verbosity >= 2: print(f"Setting {environment.alias} fallback environment.") environment.fallback = fallback_environment environment.save(using=using) except Environment.DoesNotExist: continue # Create base environment if settings.BASE_ENVIRONMENT not in aliases: alias = settings.BASE_ENVIRONMENT name = alias.title() environment = Environment() environment.alias = alias environment.name = name environment.order = -1 environment.is_active = True environment.save(using=using)
def load_label(self): """ Loads fixtures files for a given label. """ show_progress = self.verbosity >= 3 for fixture_file, fixture_dir, fixture_name in self.find_fixtures(): _, ser_fmt = self.parse_name(os.path.basename(fixture_file)) fixture = open(fixture_file, 'rb') try: self.fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if self.verbosity >= 2: self.stdout.write( "Installing %s fixture '%s' from %s." % (ser_fmt, fixture_name, humanize(fixture_dir))) objects = serializers.deserialize( ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore) for obj in objects: objects_in_fixture += 1 if router.allow_migrate_model(self.using, obj.object.__class__): loaded_objects_in_fixture += 1 self.models.add(obj.object.__class__) try: keys = dict([(key, getattr(obj.object, key)) for key in obj.object.unique_together ]) if self.update: vals = dict([(k, v) for k, v in model_to_dict( obj.object).items() if v is not None]) item, created = obj.object.__class__.objects.get_or_create( defaults=vals, **keys) if not created: if show_progress: self.stdout.write( 'Updated an object(s).', ending='') for k, v in vals.items(): setattr(item, k, v) item.save() else: if not obj.object.__class__.objects.filter( **keys).exists(): obj.object.save() if show_progress: self.stdout.write('\rProcessed %i object(s).' % loaded_objects_in_fixture, ending='') except (DatabaseError, IntegrityError) as e: e.args = ( "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'error_msg': force_text(e) }, ) raise if objects and show_progress: self.stdout.write( '') # add a newline after progress indicator self.loaded_object_count += loaded_objects_in_fixture self.fixture_object_count += objects_in_fixture except Exception as e: if not isinstance(e, CommandError): e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e), ) raise finally: fixture.close() # Warn if the fixture we loaded contains 0 objects. if objects_in_fixture == 0: warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, RuntimeWarning)
def get_objects(): for model in serializers.sort_dependencies(app_list): if (model._meta.can_migrate(self.connection) and router.allow_migrate_model(self.connection.alias, model)): queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name) yield from queryset.iterator()
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs): """ 自定义创建权限数据 """ if not app_config.models_module: return app_label = app_config.label try: app_config = apps.get_app_config(app_label) ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, Permission): return # 删除django auth创建的权限数据 Permission.objects.all().delete() # 全部的app模型 app_label_list = map( lambda item: item[0], set(ContentType.objects.all().values_list('app_label'))) # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() for app_label in app_label_list: app_config = apps.get_app_config(app_label) for klass in app_config.get_models(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("content_type", "codename")) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s'" % perm)
def update_ctypes_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs): """ Creates resources content types and default permissions. Most bits taken from contenttypes.management.update_contenttypes and auth.management.create_permissions. This will not make any attempt to clear content types no longer associated to resources. Note: ----- Because content types will not remain associated with real models, Django will ask to remove them after every migration. Although this function will recreate them, model instances with non-weak relations to them will be also deleted by the cascade, so for the time being is better to say "no" asked to remove stale content types. TODO: This *may* be addressed/mitigated defining a database router with a defined allow_migrate() method that checks the app against ContentType; e.g.: def allow_migrate(self, db, app_label, model_name=None, **hints): if <this app_label comes from a restorm app>: # don't allow the migration # (although maybe it's worth checking mode_name) return False # no opinion (let other routers decide) return None see docs.djangoproject.com/en/1.8/topics/db/multi-db/#allow_migrate """ if not isinstance(app_config, RestormAppConfig): # Any model will end up here, we are only interested in # restorm resources. return if not app_config.models_module: # This is left here for compatibility with Django. # Works because restorm resources are defined in the models.py. return try: ContentType = apps.get_model('contenttypes', 'ContentType') Permission = apps.get_model('auth', 'Permission') except LookupError: return if not router.allow_migrate_model(using, ContentType) \ or not router.allow_migrate_model(using, Permission): return ContentType.objects.clear_cache() app_label = app_config.label app_resources = { resource._meta.resource_name: resource for resource in app_config.get_resources() } if not app_resources: return # Get all the content types for this app content_types = { ct.model: ct for ct in ContentType.objects.using(using).filter(app_label=app_label) } # Create in memory any missing content type cts = [ ContentType(app_label=app_label, model=resources_name) for (resources_name, resource) in six.iteritems(app_resources) if resources_name not in content_types ] # Bulk-create the new instances ContentType.objects.using(using).bulk_create(cts) if verbosity >= 2: msg = "Adding content type '{0.app_label} | {0.model}' (restorm)".format for ct in cts: print(msg(ct)) # This will hold the permissions we're looking for as # (content_type, (codename, name)) searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() for klass in app_resources.values(): # Force looking up the content types in the current database # before creating foreign keys to them. ctype = ContentType.objects.db_manager(using).get_for_model(klass) ctypes.add(ctype) for perm in _get_all_permissions(klass._meta, ctype): searched_perms.append((ctype, perm)) # Find all the Permissions that have a content_type for a resource we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set( Permission.objects.using(using).filter( content_type__in=ctypes, ).values_list("content_type", "codename")) perms = [ Permission(codename=codename, name=name, content_type=ct) for ct, (codename, name) in searched_perms if (ct.pk, codename) not in all_perms ] # Validate the permissions before bulk_creation to avoid cryptic # database error when the verbose_name is longer than 50 characters. permission_name_max_length = Permission._meta.get_field('name').max_length verbose_name_max_length = permission_name_max_length - 11 # len('Can change ') prefix for perm in perms: if len(perm.name) > permission_name_max_length: raise exceptions.ValidationError( "The verbose_name of %s.%s is longer than %s characters" % ( perm.content_type.app_label, perm.content_type.model, verbose_name_max_length, )) Permission.objects.using(using).bulk_create(perms) if verbosity >= 2: for perm in perms: print("Adding permission '%s' (restorm)" % perm)