Ejemplo n.º 1
0
 def get_objects():
     for model in serializers.sort_dependencies(app_list):
         if (model._meta.can_migrate(self.connection) and
                 router.allow_migrate_model(self.connection.alias, model)):
             queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
             for obj in queryset.iterator():
                 yield obj
Ejemplo n.º 2
0
    def create_table(self, database, tablename, dry_run):
        cache = BaseDatabaseCache(tablename, {})
        if not router.allow_migrate_model(database, cache.cache_model_class):
            return
        connection = connections[database]

        if tablename in connection.introspection.table_names():
            if self.verbosity > 0:
                self.stdout.write("Cache table '%s' already exists." % tablename)
            return

        fields = (
            # "key" is a reserved word in MySQL, so use "cache_key" instead.
            models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True),
            models.TextField(name='value'),
            models.DateTimeField(name='expires', db_index=True),
        )
        table_output = []
        index_output = []
        qn = connection.ops.quote_name
        for f in fields:
            field_output = [qn(f.name), f.db_type(connection=connection)]
            field_output.append("%sNULL" % ("NOT " if not f.null else ""))
            if f.primary_key:
                field_output.append("PRIMARY KEY")
            elif f.unique:
                field_output.append("UNIQUE")
            if f.db_index:
                unique = "UNIQUE " if f.unique else ""
                index_output.append("CREATE %sINDEX %s ON %s (%s);" %
                    (unique, qn('%s_%s' % (tablename, f.name)), qn(tablename),
                    qn(f.name)))
            table_output.append(" ".join(field_output))
        full_statement = ["CREATE TABLE %s (" % qn(tablename)]
        for i, line in enumerate(table_output):
            full_statement.append('    %s%s' % (line, ',' if i < len(table_output) - 1 else ''))
        full_statement.append(');')

        full_statement = "\n".join(full_statement)

        if dry_run:
            self.stdout.write(full_statement)
            for statement in index_output:
                self.stdout.write(statement)
            return

        with transaction.atomic(using=database,
                                savepoint=connection.features.can_rollback_ddl):
            with connection.cursor() as curs:
                try:
                    curs.execute(full_statement)
                except DatabaseError as e:
                    raise CommandError(
                        "Cache table '%s' could not be created.\nThe error was: %s." %
                        (tablename, force_text(e)))
                for statement in index_output:
                    curs.execute(statement)

        if self.verbosity > 1:
            self.stdout.write("Cache table '%s' created." % tablename)
Ejemplo n.º 3
0
def db_router_allows_migrate(database, app_label, model_cls):
    """Return whether a database router allows migrate operations for a model.

    This will only return ``True`` for Django 1.7 and newer and if the
    router allows migrate operations. This is compatible with both the
    Django 1.7 and 1.8+ versions of ``allow_migrate``.

    Args:
        database (unicode):
            The name of the database.

        app_label (unicode):
            The application label.

        model_cls (type):
            The model class.

    Returns:
        bool:
        ``True`` if routers allow migrate for this model.
    """
    if django.VERSION[:2] >= (1, 8):
        return router.allow_migrate_model(database, model_cls)
    elif django.VERSION[:2] == (1, 7):
        return router.allow_migrate(database, model_cls)
    else:
        return False
Ejemplo n.º 4
0
def save_all(objects, using, iterations=0, stdout=None):
    '''Tries recursively saves all objects'''

    if len(objects) == 0 or iterations >= 25:
        return iterations

    skiped = []

    for obj in objects:
        if router.allow_migrate_model(using, obj.object.__class__):
            try:
                obj.save(using=using)
            except (DatabaseError, IntegrityError) as e:
                e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
                    'app_label': obj.object._meta.app_label,
                    'object_name': obj.object._meta.object_name,
                    'pk': obj.object.pk,
                    'error_msg': force_text(e)
                },)
                # raise
                skiped.append(obj)

    if len(skiped) > 0:
        return save_all(skiped, using, iterations + 1)

    return iterations
Ejemplo n.º 5
0
def load_data_for_db(db_alias, objects):
    """
    :param db_alias: Django alias for database to load objects into
    :param objects: List of object dictionaries to load
    :return: LoadStats object
    """
    connection = connections[db_alias]

    model_counter = Counter()
    with connection.constraint_checks_disabled():
        for obj in PythonDeserializer(objects, using=db_alias):
            if router.allow_migrate_model(db_alias, obj.object.__class__):
                model_counter.update([obj.object.__class__])
                try:
                    obj.save(using=db_alias)
                except (DatabaseError, IntegrityError) as e:
                    e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
                        'app_label': obj.object._meta.app_label,
                        'object_name': obj.object._meta.object_name,
                        'pk': obj.object.pk,
                        'error_msg': force_text(e)
                    },)
                    raise

    # Since we disabled constraint checks, we must manually check for
    # any invalid keys that might have been added
    table_names = [model._meta.db_table for model in model_counter]
    try:
        connection.check_constraints(table_names=table_names)
    except Exception as e:
        e.args = ("Problem loading data: %s" % e,)
        raise

    return LoadStat(db_alias, model_counter)
Ejemplo n.º 6
0
        def get_objects(count_only=False):
            """
            Collate the objects to be serialized. If count_only is True, just
            count the number of objects to be serialized.
            """
            models = serializers.sort_dependencies(app_list.items())
            for model in models:
                if model in excluded_models:
                    continue
                if model._meta.proxy and model._meta.proxy_for_model not in models:
                    warnings.warn(
                        "%s is a proxy model and won't be serialized." % model._meta.label,
                        category=ProxyModelWarning,
                    )
                if not model._meta.proxy and router.allow_migrate_model(using, model):
                    if use_base_manager:
                        objects = model._base_manager
                    else:
                        objects = model._default_manager

                    queryset = objects.using(using).order_by(model._meta.pk.name)
                    if primary_keys:
                        queryset = queryset.filter(pk__in=primary_keys)
                    if count_only:
                        yield queryset.order_by().count()
                    else:
                        for obj in queryset.iterator():
                            yield obj
Ejemplo n.º 7
0
def create_default_site(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
    try:
        Site = apps.get_model('sites', 'Site')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Site):
        return

    if not Site.objects.using(using).exists():
        # The default settings set SITE_ID = 1, and some tests in Django's test
        # suite rely on this value. However, if database sequences are reused
        # (e.g. in the test suite after flush/syncdb), it isn't guaranteed that
        # the next id will be 1, so we coerce it. See #15573 and #16353. This
        # can also crop up outside of tests - see #15346.
        if verbosity >= 2:
            print("Creating example.com Site object")
        Site(pk=settings.SITE_ID, domain="example.com", name="example.com").save(using=using)

        # We set an explicit pk instead of relying on auto-incrementation,
        # so we need to reset the database sequence. See #17415.
        sequence_sql = connections[using].ops.sequence_reset_sql(no_style(), [Site])
        if sequence_sql:
            if verbosity >= 2:
                print("Resetting sequence")
            with connections[using].cursor() as cursor:
                for command in sequence_sql:
                    cursor.execute(command)
Ejemplo n.º 8
0
def load_cloud_objects(app_config, verbosity=2, interactive=True,
                       using=DEFAULT_DB_ALIAS, **kwargs):

    if not app_config.models_module:
        return

    initial_data = load_initial_data()

    for model in initial_data:
        logger.info("Attempting to load data for {}...".format(model['model']))
        # Grab the model class, but don't fail if we can't find it
        try:
            model_cls = apps.get_model('cloud', model['model'])
        except LookupError:
            logger.warning('Failed to load model class: {}'.format(model['model']))
            continue

        # If we can't migrate this model, don't do anything & go on to the next
        if not router.allow_migrate_model(using, model_cls):
            logger.info("Skipping data load for {}".format(model['model']))
            continue

        to_create = []

        # Grab the attribute to filter on
        filter_attr = model['filter_attr']

        for object_data in model['objects']:
            # Only create if it's not already there
            filtered = model_cls.objects.filter(**{filter_attr: object_data[filter_attr]})

            if filtered.count() == 0:
                # Object doesn't exist in the database
                if 'pk' in object_data:
                    try:
                        model_cls.objects.get(pk=object_data['pk'])

                        # This means there's a conflicting pk, so we'll just remove the pk from
                        # the object_data dict and let it get assigned something else.
                        del object_data['pk']
                    except exceptions.ObjectDoesNotExist:
                        # There's no conflicting object, this is good
                        pass

                # Add this object to the create list
                to_create.append(model_cls(**object_data))
            else:
                # This object already exists in the database
                # Update the fields to match the new stuff if the object already exists
                for obj in filtered:
                    for attr, val in object_data.items():
                        # We don't want to change the object's primary key
                        #   it would mess up a bunch of relations
                        if attr != 'pk':
                            setattr(obj, attr, val)
                    obj.save()

        # bulk create everything
        model_cls.objects.using(using).bulk_create(to_create)
Ejemplo n.º 9
0
    def load_label(self, fixture_label):
        """
        Loads fixtures files for a given label.
        """
        show_progress = self.verbosity >= 3
        for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label):
            _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
            open_method, mode = self.compression_formats[cmp_fmt]
            fixture = open_method(fixture_file, mode)
            try:
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write("Installing %s fixture '%s' from %s." %
                        (ser_fmt, fixture_name, humanize(fixture_dir)))

                objects = serializers.deserialize(ser_fmt, fixture,
                    using=self.using, ignorenonexistent=self.ignore)

                for obj in objects:
                    objects_in_fixture += 1
                    if router.allow_migrate_model(self.using, obj.object.__class__):
                        loaded_objects_in_fixture += 1
                        self.models.add(obj.object.__class__)
                        try:
                            obj.save(using=self.using)
                            if show_progress:
                                self.stdout.write(
                                    '\rProcessed %i object(s).' % loaded_objects_in_fixture,
                                    ending=''
                                )
                        except (DatabaseError, IntegrityError) as e:
                            e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
                                'app_label': obj.object._meta.app_label,
                                'object_name': obj.object._meta.object_name,
                                'pk': obj.object.pk,
                                'error_msg': force_text(e)
                            },)
                            raise
                if objects and show_progress:
                    self.stdout.write('')  # add a newline after progress indicator
                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                if not isinstance(e, CommandError):
                    e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),)
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture_name,
                    RuntimeWarning
                )
Ejemplo n.º 10
0
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
    if not app_config.models_module:
        return

    try:
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    from django.contrib.contenttypes.models import ContentType

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(klass)
        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta, ctype):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(Permission.objects.using(using).filter(
        content_type__in=ctypes,
    ).values_list(
        "content_type", "codename"
    ))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    # Validate the permissions before bulk_creation to avoid cryptic
    # database error when the verbose_name is longer than 50 characters
    permission_name_max_length = Permission._meta.get_field('name').max_length
    verbose_name_max_length = permission_name_max_length - 11  # len('Can change ') prefix
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            raise exceptions.ValidationError(
                "The verbose_name of %s.%s is longer than %s characters" % (
                    perm.content_type.app_label,
                    perm.content_type.model,
                    verbose_name_max_length,
                )
            )
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 11
0
    def allow_migrate_model(self, connection_alias, model):
        """
        Returns if we're allowed to migrate the model.

        This is a thin wrapper around router.allow_migrate_model() that
        preemptively rejects any proxy, swapped out, or unmanaged model.
        """
        if not model._meta.can_migrate(connection_alias):
            return False

        return router.allow_migrate_model(connection_alias, model)
Ejemplo n.º 12
0
    def load_label(self, fixture_label, ser_fmt='json'):
        """
        Uses get_latest_fixture to load fixture data
        """

        with get_latest_fixture(fixture_label) as fixture:
            try:
                self.fixture_name = fixture.name
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write("Installing %s fixture." % (
                        fixture.name
                    ))

                objects = serializers.deserialize(
                    ser_fmt, fixture, using=self.using,
                    ignorenonexistent=self.ignore
                )

                for obj in objects:
                    objects_in_fixture += 1
                    if router.allow_migrate_model(self.using,
                                                  obj.object.__class__):
                        loaded_objects_in_fixture += 1
                        self.models.add(obj.object.__class__)
                        try:
                            obj.save(using=self.using)
                        except (DatabaseError, IntegrityError) as e:
                            e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
                                'app_label': obj.object._meta.app_label,
                                'object_name': obj.object._meta.object_name,
                                'pk': obj.object.pk,
                                'error_msg': force_text(e)
                            },)
                            raise

                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                e.args = ("Problem installing fixture '%s': %s" % (fixture.name, e),)
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture.name,
                    RuntimeWarning
                )
Ejemplo n.º 13
0
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs):
    if not app_config.models_module:
        return

    # Ensure that contenttypes are created for this app. Needed if
    # 'django.contrib.auth' is in INSTALLED_APPS before
    # 'django.contrib.contenttypes'.
    create_contenttypes(app_config, verbosity=verbosity, interactive=interactive, using=using, apps=apps, **kwargs)

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = []
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(klass, for_concrete_model=False)

        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(Permission.objects.using(using).filter(
        content_type__in=ctypes,
    ).values_list(
        "content_type", "codename"
    ))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 14
0
def get_contenttypes_and_models(app_config, using, ContentType):
    if not router.allow_migrate_model(using, ContentType):
        return None, None

    ContentType.objects.clear_cache()

    content_types = {
        ct.model: ct
        for ct in ContentType.objects.using(using).filter(app_label=app_config.label)
    }
    app_models = {
        model._meta.model_name: model
        for model in app_config.get_models()
    }
    return content_types, app_models
Ejemplo n.º 15
0
def get_all_model_querysets_for_domain(model_class, domain):
    using = router.db_for_read(model_class)
    if settings.USE_PARTITIONED_DATABASE and using == partition_config.get_proxy_db():
        using = partition_config.get_form_processing_dbs()
    else:
        using = [using]

    for db_alias in using:
        if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class):
            objects = model_class._default_manager

            queryset = objects.using(db_alias).order_by(model_class._meta.pk.name)

            filters = get_model_domain_filters(model_class, domain)
            for filter in filters:
                yield model_class, queryset.filter(filter)
Ejemplo n.º 16
0
        def get_objects():
            # Collate the objects to be serialized.
            for model in serializers.sort_dependencies(app_list.items()):
                if model in excluded_models:
                    continue
                if not model._meta.proxy and router.allow_migrate_model(using, model):
                    if use_base_manager:
                        objects = model._base_manager
                    else:
                        objects = model._default_manager

                    queryset = objects.using(using).order_by(model._meta.pk.name)
                    if primary_keys:
                        queryset = queryset.filter(pk__in=primary_keys)
                    for obj in queryset.iterator():
                        yield obj
Ejemplo n.º 17
0
def get_all_model_iterators_builders_for_domain(model_class, domain, limit_to_db=None):
    using = router.db_for_read(model_class)
    if settings.USE_PARTITIONED_DATABASE and using == partition_config.get_proxy_db():
        using = partition_config.get_form_processing_dbs()
    else:
        using = [using]

    if limit_to_db:
        if limit_to_db not in using:
            raise DomainDumpError('DB specified is not valide for '
                                  'model class: {} not in {}'.format(limit_to_db, using))
        using = [limit_to_db]

    for db_alias in using:
        if not model_class._meta.proxy and router.allow_migrate_model(db_alias, model_class):
            iterator_builder = APP_LABELS_WITH_FILTER_KWARGS_TO_DUMP[get_model_label(model_class)]
            yield model_class, iterator_builder.build(domain, model_class, db_alias)
Ejemplo n.º 18
0
def create_default_site_relations(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps,
                                  **kwargs):
    try:
        Site = apps.get_model('sites', 'Site')
        SiteResources = apps.get_model('geosites', 'SiteResources')
        SitePeople = apps.get_model('geosites', 'SitePeople')
        SiteGroups = apps.get_model('geosites', 'SiteGroups')
    except LookupError:
        return

    if not router.allow_migrate_model(using, SiteResources):
        return

    for site in Site.objects.all():
        SiteResources.objects.get_or_create(site=site)
        SitePeople.objects.get_or_create(site=site)
        SiteGroups.objects.get_or_create(site=site)
Ejemplo n.º 19
0
def update_default_site(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS,
                        **kwargs):
    try:
        site_model = apps.get_model('sites', 'Site')
    except LookupError:
        return

    if not router.allow_migrate_model(using, site_model):
        return

    if verbosity >= 2:
        print('Updating example.com Site object')

    if getattr(settings, 'SITE_ID', None):
        site = site_model.objects.get(pk=settings.SITE_ID)
        site.domain = settings.SITE_DOMAIN
        site.name = settings.SITE_NAME
        site.save(using=using)
Ejemplo n.º 20
0
def create_permissions(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
    """Create permissions on django 1.10+."""
    if not app_config.models_module:
        return

    try:
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    searched_perms = list()  # (content_type, (codename, name))
    ctypes = set()           # The codenames and ctypes that should exist.

    for klass in app_config.get_models():
        ctype = ContentType.objects.db_manager(using).get_for_model(klass)
        ctypes.add(ctype)

        from django.contrib.auth.management import _get_all_permissions
        for perm in _get_all_permissions(klass._meta):
            searched_perms.append((ctype, perm))

    all_perms = set(Permission.objects.using(using).filter(
        content_type__in=ctypes,
    ).values_list(
        "content_type", "codename"
    ))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]

    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 21
0
def inject_rename_contenttypes_operations(plan=None, apps=global_apps, using=DEFAULT_DB_ALIAS, **kwargs):
    """
    Insert a `RenameContentType` operation after every planned `RenameModel`
    operation.
    """
    if plan is None:
        return

    # Determine whether or not the ContentType model is available.
    try:
        ContentType = apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        available = False
    else:
        if not router.allow_migrate_model(using, ContentType):
            return
        available = True

    for migration, backward in plan:
        if ((migration.app_label, migration.name) == ('contenttypes', '0001_initial')):
            # There's no point in going forward if the initial contenttypes
            # migration is unapplied as the ContentType model will be
            # unavailable from this point.
            if backward:
                break
            else:
                available = True
                continue
        # The ContentType model is not available yet.
        if not available:
            continue
        inserts = []
        for index, operation in enumerate(migration.operations):
            if isinstance(operation, migrations.RenameModel):
                operation = RenameContentType(
                    migration.app_label, operation.old_name_lower, operation.new_name_lower
                )
                inserts.append((index + 1, operation))
        for inserted, (index, operation) in enumerate(inserts):
            migration.operations.insert(inserted + index, operation)
Ejemplo n.º 22
0
    def _precompile_fixture(self, fixture, db):
        """Precompile a fixture.

        The fixture is loaded and deserialized, and the resulting objects
        are stored for future use.

        Args:
            fixture (unicode):
                The name of the fixture.

            db (unicode):
                The database name to load fixture data on.
        """
        assert db in self._precompiled_fixtures
        assert fixture not in self._precompiled_fixtures[db]

        fixture_path = None

        for fixture_dir in self._get_fixture_dirs():
            fixture_path = os.path.join(fixture_dir, fixture + '.json')

            if os.path.exists(fixture_path):
                break

        try:
            if not fixture_path:
                raise IOError('Fixture path not found')

            with open(fixture_path, 'r') as fp:
                self._precompiled_fixtures[db][fixture] = [
                    obj
                    for obj in serializers.deserialize('json', fp, using=db)
                    if ((hasattr(router, 'allow_syncdb') and
                         router.allow_syncdb(db, obj.object.__class__)) or
                        (hasattr(router, 'allow_migrate_model') and
                         router.allow_migrate_model(db, obj.object)))
                ]
        except IOError as e:
            sys.stderr.write('Unable to load fixture %s: %s\n' % (fixture, e))
Ejemplo n.º 23
0
def create_app_sig(app, database):
    """
    Creates a dictionary representation of the models in a given app.
    Only those attributes that are interesting from a schema-evolution
    perspective are included.
    """
    app_sig = OrderedDict()

    for model in get_models(app):
        # Only include those models that can be synced.
        #
        # On Django 1.7 and up, we need to check if the model allows for
        # migrations (using allow_migrate_model).
        #
        # On older versions of Django, we check if the model allows for
        # synchronization to the database (allow_syncdb).
        if ((hasattr(router, 'allow_syncdb') and
             router.allow_syncdb(database, model.__class__)) or
            (hasattr(router, 'allow_migrate_model') and
             router.allow_migrate_model(database, model))):
            app_sig[model._meta.object_name] = create_model_sig(model)

    return app_sig
Ejemplo n.º 24
0
    def _rename(self, apps, schema_editor, old_model, new_model):
        ContentType = apps.get_model('contenttypes', 'ContentType')
        db = schema_editor.connection.alias
        if not router.allow_migrate_model(db, ContentType):
            return

        try:
            content_type = ContentType.objects.db_manager(db).get_by_natural_key(self.app_label, old_model)
        except ContentType.DoesNotExist:
            pass
        else:
            content_type.model = new_model
            try:
                with transaction.atomic(using=db):
                    content_type.save(update_fields={'model'})
            except IntegrityError:
                # Gracefully fallback if a stale content type causes a
                # conflict as update_contenttypes will take care of asking the
                # user what should be done next.
                content_type.model = old_model
            else:
                # Clear the cache as the `get_by_natual_key()` call will cache
                # the renamed ContentType instance by its old model name.
                ContentType.objects.clear_cache()
Ejemplo n.º 25
0
    def load_label(self, fixture_label):
        """Load fixtures files for a given label."""
        show_progress = self.verbosity >= 3
        for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label):
            _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
            open_method, mode = self.compression_formats[cmp_fmt]
            fixture = open_method(fixture_file, mode)
            try:
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write(
                        "Installing %s fixture '%s' from %s."
                        % (ser_fmt, fixture_name, humanize(fixture_dir))
                    )

                objects = serializers.deserialize(
                    ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore,
                )

                for obj in objects:
                    objects_in_fixture += 1
                    if (obj.object._meta.app_config in self.excluded_apps or
                            type(obj.object) in self.excluded_models):
                        continue
                    if router.allow_migrate_model(self.using, obj.object.__class__):
                        loaded_objects_in_fixture += 1
                        model = obj.object.__class__
                        self.models.add(model)
                        try:
                            obj.object.validate_unique()
                        except ValidationError as e:
                            if hasattr(model.objects, "get_natural_fields"):
                                fields = model.objects.get_natural_fields()
                            else:
                                fields = e.message_dict.keys()
                            existing_object = model.objects.get(**{k: getattr(obj.object, k) for k in fields})
                            obj.object.pk = existing_object.pk
                        try:
                            obj.save(using=self.using)
                            if show_progress:
                                self.stdout.write(
                                    '\rProcessed %i object(s).' % loaded_objects_in_fixture,
                                    ending=''
                                )
                        # psycopg2 raises ValueError if data contains NUL chars.
                        except (DatabaseError, IntegrityError, ValueError) as e:
                            e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
                                'app_label': obj.object._meta.app_label,
                                'object_name': obj.object._meta.object_name,
                                'pk': obj.object.pk,
                                'error_msg': e,
                            },)
                            raise
                if objects and show_progress:
                    self.stdout.write('')  # add a newline after progress indicator
                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                if not isinstance(e, CommandError):
                    e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),)
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture_name,
                    RuntimeWarning
                )
def update_roles_permissions(
    Role, Permission, RolePermission, app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs
):
    if not router.allow_migrate_model(using, Role):
        return

    try:
        Role = app_config.get_model("trusts", "Role")
    except LookupError:
        return

    # This will hold the roles we're looking for as
    # (rolename (permissions))
    model_roles = {}
    for klass in app_config.get_models():
        if hasattr(klass._meta, "roles"):
            _process_roles(Permission, model_roles, klass, klass._meta.roles, using)
        elif hasattr(klass._meta, "content_roles"):
            if hasattr(klass, "get_content_model"):
                content_klass = klass.get_content_model()
                _process_roles(Permission, model_roles, content_klass, klass._meta.content_roles, using)

    # Find all the Roles and its Permission
    db_roles = {}
    for r in Role.objects.using(using).all():
        db_roles[r.name] = set(r.permissions.all())

    # Get all the diff between sets
    model_rolenames = set(model_roles.keys())
    db_rolenames = set(db_roles.keys())
    added_rolenames = model_rolenames - db_rolenames
    deleted_rolenames = db_rolenames - model_rolenames
    existing_rolenames = model_rolenames.intersection(db_rolenames)

    # Prepare rolepermissions for bulk op at the end
    bulk_add_rolepermissions = []
    q_del_rolepermissions = []
    deleted_role_ids = []
    # Process added roles
    for rolename in added_rolenames:
        r = Role(name=rolename)
        r.save()
        for p in model_roles[rolename]:
            bulk_add_rolepermissions.append(RolePermission(managed=True, permission=p, role=r))

    # Process existing roles
    for rolename in existing_rolenames:
        r = Role.objects.get(name=rolename)
        db_permissions = db_roles[rolename]
        model_permissions = model_roles[rolename]

        added_permissions = set(model_permissions) - set(db_permissions)
        for p in added_permissions:
            bulk_add_rolepermissions.append(RolePermission(managed=True, permission=p, role=r))

        deleted_permissions = set(db_permissions) - set(model_permissions)
        if len(deleted_permissions):
            q_del_rolepermissions.append((r, Q(managed=True, role=r, permission__in=deleted_permissions)))

    # Process deleted roles
    for rolename in deleted_rolenames:
        r = Role.objects.get(name=rolename)
        q_del_rolepermissions.append((r, Q(managed=True, role=r)))
        deleted_role_ids.append(r.pk)

    # Create the added role permissions
    RolePermission.objects.using(using).bulk_create(bulk_add_rolepermissions)
    if verbosity >= 2:
        for rolepermission in bulk_add_rolepermissions:
            print('Adding role(%s).rolepermission "%s"' % (rolepermission.role.name, rolepermission))

    # Remove the deleted role permissions
    for r, q in q_del_rolepermissions:
        qs = RolePermission.objects.filter(q)
        if verbosity >= 2:
            if qs.count() > 0:
                for rolepermission in qs.all():
                    print('Removing role(%s).rolepermission "%s"' % (rolepermission.role.name, rolepermission))
        qs.delete()

    # Remove the deleted role
    qs = Role.objects.filter(pk__in=deleted_role_ids, permissions__isnull=True)
    if verbosity >= 2:
        if qs.count() > 0:
            for role in qs.all():
                print('Removing role "%s"' % (role.name))
    qs.delete()
Ejemplo n.º 27
0
def create_permissions(app_config,
                       verbosity=2,
                       interactive=True,
                       using=DEFAULT_DB_ALIAS,
                       **kwargs):
    """
    Copy/paste from django/contrib/auth/management/__init__.py with
    small modifications (for_concrete_model, app_config.get_proxies).

    Create permissions for defined proxy models in apps module.
    """
    from django.apps import apps

    if not app_config.models_module:
        return

    try:
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    from django.contrib.contenttypes.models import ContentType

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(
            model=klass, for_concrete_model=False)
        if klass._meta.proxy:
            concrete_ctype = ContentType.objects.db_manager(
                using).get_for_model(  # noqa
                    model=klass, )
            perms = Permission.objects.using(using).filter(
                content_type=concrete_ctype,
                codename__endswith=klass._meta.model_name)
            if perms:
                perms.update(content_type=ctype)
        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta, ctype):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("content_type", "codename"))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    # Validate the permissions before bulk_creation to avoid cryptic
    # database error when the verbose_name is longer than 50 characters
    permission_name_max_length = Permission._meta.get_field('name').max_length
    verbose_name_max_length = permission_name_max_length - len('Can change ')
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            raise exceptions.ValidationError(
                "The verbose_name of %s.%s is longer than %s characters" % (
                    perm.content_type.app_label,
                    perm.content_type.model,
                    verbose_name_max_length,
                ))
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 28
0
def update_contenttypes(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs):
    """
    Creates content types for models in the given app, removing any model
    entries that no longer have a matching model class.
    """
    if not app_config.models_module:
        return

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        return

    if not router.allow_migrate_model(using, ContentType):
        return

    ContentType.objects.clear_cache()
    # Always clear the global content types cache.
    if apps is not global_apps:
        global_apps.get_model('contenttypes', 'ContentType').objects.clear_cache()

    app_models = {
        model._meta.model_name: model
        for model in app_config.get_models()}

    if not app_models:
        return

    # Get all the content types
    content_types = {
        ct.model: ct
        for ct in ContentType.objects.using(using).filter(app_label=app_label)
    }
    to_remove = [
        ct
        for (model_name, ct) in six.iteritems(content_types)
        if model_name not in app_models
    ]

    cts = [
        ContentType(
            app_label=app_label,
            model=model_name,
        )
        for (model_name, model) in six.iteritems(app_models)
        if model_name not in content_types
    ]
    ContentType.objects.using(using).bulk_create(cts)
    if verbosity >= 2:
        for ct in cts:
            print("Adding content type '%s | %s'" % (ct.app_label, ct.model))

    # Confirm that the content type is stale before deletion.
    using = router.db_for_write(ContentType)
    if to_remove:
        if interactive:
            ct_info = []
            for ct in to_remove:
                ct_info.append('    - Content type for %s.%s' % (ct.app_label, ct.model))
                collector = NoFastDeleteCollector(using=using)
                collector.collect([ct])

                for obj_type, objs in collector.data.items():
                    if objs == {ct}:
                        continue
                    ct_info.append('    - %s object%s of type %s.%s:' % (
                        len(objs),
                        's' if len(objs) != 1 else '',
                        obj_type._meta.app_label,
                        obj_type._meta.model_name)
                    )

            content_type_display = '\n'.join(ct_info)
            print("""Some content types in your database are stale and can be deleted.
Any objects that depend on these content types will then also be deleted.
The content types, and the dependent objects that would be deleted, are:

%s

This list does not include data that might be in your database
outside of Django's models.

Are you sure you want to delete these content types?
If you're unsure, answer 'no'.
    """ % content_type_display)
            ok_to_delete = input("Type 'yes' to continue, or 'no' to cancel: ")
        else:
            ok_to_delete = False

        if ok_to_delete == 'yes':
            for ct in to_remove:
                if verbosity >= 2:
                    print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model))
                ct.delete()
        else:
            if verbosity >= 2:
                print("Stale content types remain.")
    def load_label(self, fixture_label):
        """
        Loads fixtures files for a given label. This method is largely copied
        from django.core.management.commands.loaddata.Command but with the
        addition of using bulk_create where possible.
        """
        show_progress = self.verbosity >= 3
        for fixture_file, fixture_dir, fixture_name in self.find_fixtures(
                fixture_label):
            _, ser_fmt, cmp_fmt = self.parse_name(
                os.path.basename(fixture_file))
            open_method, mode = self.compression_formats[cmp_fmt]
            fixture = open_method(fixture_file, mode)
            try:
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write(
                        "Installing %s fixture '%s' from %s." %
                        (ser_fmt, fixture_name, humanize(fixture_dir)))

                objects = serializers.deserialize(
                    ser_fmt,
                    fixture,
                    using=self.using,
                    ignorenonexistent=self.ignore,
                )

                models_in_file = set()
                objects_to_create = []
                for obj in objects:
                    objects_in_fixture += 1
                    if (obj.object._meta.app_config in self.excluded_apps
                            or type(obj.object) in self.excluded_models):
                        continue
                    if router.allow_migrate_model(self.using,
                                                  obj.object.__class__):
                        self.models.add(obj.object.__class__)
                        models_in_file.add(obj.object.__class__)
                        objects_to_create.append(obj)

                if len(models_in_file) == 1:
                    model = list(models_in_file)[0]
                    try:
                        model_objects = []
                        m2m_related_objects = defaultdict(list)
                        for obj in objects_to_create:
                            model_objects.append(obj.object)
                            if obj.m2m_data:
                                for accessor_field, m2m_list in obj.m2m_data.items(
                                ):
                                    m2m_rel = getattr(model,
                                                      accessor_field).rel
                                    related_model = m2m_rel.model
                                    through_model = m2m_rel.through

                                    for m2m_id in m2m_list:
                                        model_id_field = '{}_id'.format(
                                            model._meta.model_name)
                                        related_model_id_field = '{}_id'.format(
                                            related_model._meta.model_name)
                                        m2m_related_objects[
                                            through_model].append(
                                                through_model(
                                                    **{
                                                        model_id_field:
                                                        obj.object.pk,
                                                        related_model_id_field:
                                                        m2m_id
                                                    }))

                        model.objects.bulk_create(model_objects)
                        for m2m_model in m2m_related_objects:
                            m2m_model.objects.bulk_create(
                                m2m_related_objects[m2m_model])
                        loaded_objects_in_fixture += len(objects_to_create)
                        if show_progress:
                            self.stdout.write('\rProcessed %i object(s).' %
                                              loaded_objects_in_fixture,
                                              ending='')
                    except (DatabaseError, IntegrityError) as e:
                        e.args = (
                            "Could not load %(app_label)s.%(object_name)s: %(error_msg)s"
                            % {
                                'app_label': model._meta.app_label,
                                'object_name': model._meta.object_name,
                                'error_msg': force_text(e)
                            }, )
                        raise
                else:
                    for obj in objects_to_create:
                        try:
                            obj.save(using=self.using)
                            loaded_objects_in_fixture += 1
                            if show_progress:
                                self.stdout.write('\rProcessed %i object(s).' %
                                                  loaded_objects_in_fixture,
                                                  ending='')
                        except (DatabaseError, IntegrityError) as e:
                            e.args = (
                                "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s"
                                % {
                                    'app_label': obj.object._meta.app_label,
                                    'object_name':
                                    obj.object._meta.object_name,
                                    'pk': obj.object.pk,
                                    'error_msg': force_text(e)
                                }, )
                            raise
                if objects and show_progress:
                    self.stdout.write(
                        '')  # add a newline after progress indicator
                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                if not isinstance(e, CommandError):
                    e.args = ("Problem installing fixture '%s': %s" %
                              (fixture_file, e), )
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture_name, RuntimeWarning)
    def create_application_dynamic_permissions(self,
                                               app_config,
                                               db_alias=DEFAULT_DB_ALIAS,
                                               **kwargs):
        from collections import OrderedDict
        from django.apps import apps
        from django.core.exceptions import ImproperlyConfigured
        from django.db import IntegrityError
        from django.db import router

        try:
            ContentType = apps.get_model('contenttypes', 'ContentType')
            ModelAction = apps.get_model('talos', 'ModelAction')
            ObjectAction = apps.get_model('talos', 'ObjectAction')
        except LookupError:
            return

        if not router.allow_migrate_model(db_alias, ModelAction):
            return

        if not router.allow_migrate_model(db_alias, ObjectAction):
            return

        model_action_dict = OrderedDict()
        object_action_dict = OrderedDict()

        for model in app_config.get_models():
            if hasattr(model._meta, '_talos_model_info'):
                model_info = model._meta._talos_model_info

                if model_info.has_actions():
                    content_type = ContentType.objects.db_manager(
                        db_alias).get_for_model(model_info.model)

                    for action_info in model_info.model_action_infos.values():
                        model_action_dict[action_info.code] = ModelAction(
                            application=model._meta.app_config.label,
                            model=model._meta.label_lower,
                            action=action_info.action,
                            code=action_info.code,
                            name=action_info.name,
                            content_type=content_type)

                    for action_info in model_info.object_action_infos.values():
                        object_action_dict[action_info.code] = ObjectAction(
                            application=model._meta.app_config.label,
                            model=model._meta.label_lower,
                            action=action_info.action,
                            code=action_info.code,
                            name=action_info.name,
                            content_type=content_type)

        existing_model_action_codes = list(
            ModelAction.objects.using(db_alias).filter(
                code__in=model_action_dict.keys()).values_list('code',
                                                               flat=True))
        existing_object_action_codes = list(
            ObjectAction.objects.using(db_alias).filter(
                code__in=object_action_dict.keys()).values_list('code',
                                                                flat=True))

        missing_model_action_dict = {
            code: model_action_dict[code]
            for code in model_action_dict
            if code not in existing_model_action_codes
        }
        missing_object_action_dict = {
            code: object_action_dict[code]
            for code in object_action_dict
            if code not in existing_object_action_codes
        }

        try:
            ModelAction.objects.using(db_alias).bulk_create(
                missing_model_action_dict.values())
        except IntegrityError:
            for model_action in missing_model_action_dict.values():
                try:
                    model_action.save()
                except IntegrityError:
                    raise ImproperlyConfigured(
                        INVALID_PERMISSION_NAME_MESSAGE.format(
                            model_action.name))

        try:
            ObjectAction.objects.using(db_alias).bulk_create(
                missing_object_action_dict.values())
        except IntegrityError:
            for object_action in missing_object_action_dict.values():
                try:
                    object_action.save()
                except IntegrityError:
                    raise ImproperlyConfigured(
                        INVALID_PERMISSION_NAME_MESSAGE.format(
                            object_action.name))
Ejemplo n.º 31
0
def update_contenttypes(sender, verbosity=2, db=DEFAULT_DB_ALIAS, **kwargs):
    """
        Django's default update_contenttypes relies on many inconsistent queries which causes problems
        with syncdb. This monkeypatch replaces it with a version that does look ups on unique constraints
        which are slightly better protected from eventual consistency issues by the context cache.
    """
    from django.contrib.contenttypes.models import ContentType
    
    if verbosity >= 2:
        print("Running Djangae version of update_contenttypes on {}".format(sender))

    try:
        apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        return

    if hasattr(router, "allow_migrate_model"):
        if not router.allow_migrate_model(db, ContentType):
            return

    ContentType.objects.clear_cache()
    app_models = sender.get_models()
    if not app_models:
        return
    # They all have the same app_label, get the first one.
    app_label = sender.label
    app_models = dict(
        (model._meta.model_name, model)
        for model in app_models
    )

    created_or_existing_pks = []
    created_or_existing_by_unique = {}

    for (model_name, model) in app_models.items():
        # Go through get_or_create any models that we want to keep
        defaults = {}
        if django.VERSION < (1, 9):
            defaults['name'] = smart_text(model._meta.verbose_name_raw)

        ct, created = ContentType.objects.get_or_create(
            app_label=app_label,
            model=model_name,
            defaults=defaults,
        )

        if verbosity >= 2 and created:
            print("Adding content type '%s | %s'" % (ct.app_label, ct.model))

        created_or_existing_pks.append(ct.pk)
        created_or_existing_by_unique[(app_label, model_name)] = ct.pk

    # Now lets see if we should remove any

    to_remove = [x for x in ContentType.objects.filter(app_label=app_label) if x.pk not in created_or_existing_pks]

    # Now it's possible that our get_or_create failed because of consistency issues and we create a duplicate.
    # Then the original appears in the to_remove and we remove the original. This is bad. So here we go through the
    # to_remove list, and if we created the content type just now, we delete that one, and restore the original in the
    # cache
    for ct in to_remove:
        unique = (ct.app_label, ct.model)
        if unique in created_or_existing_by_unique:
            # We accidentally created a duplicate above due to HRD issues, delete the one we created
            ContentType.objects.get(pk=created_or_existing_by_unique[unique]).delete()
            created_or_existing_by_unique[unique] = ct.pk
            ct.save()  # Recache this one in the context cache

    to_remove = [ x for x in to_remove if (x.app_label, x.model) not in created_or_existing_by_unique ]

    # Now, anything left should actually be a stale thing. It's still possible we missed some but they'll get picked up
    # next time. Confirm that the content type is stale before deletion.
    if to_remove:
        if kwargs.get('interactive', False):
            content_type_display = '\n'.join([
                '    %s | %s' % (x.app_label, x.model)
                for x in to_remove
            ])
            ok_to_delete = input("""The following content types are stale and need to be deleted:

%s

Any objects related to these content types by a foreign key will also
be deleted. Are you sure you want to delete these content types?
If you're unsure, answer 'no'.

    Type 'yes' to continue, or 'no' to cancel: """ % content_type_display)
        else:
            ok_to_delete = False

        if ok_to_delete == 'yes':
            for ct in to_remove:
                if verbosity >= 2:
                    print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model))
                ct.delete()
        else:
            if verbosity >= 2:
                print("Stale content types remain.")
Ejemplo n.º 32
0
def allow_migrate(model):
    for db in connections:
        if router.allow_migrate_model(db, model):
            yield db
Ejemplo n.º 33
0
def create_report_permissions(app_config,
                              verbosity=2,
                              interactive=True,
                              using=DEFAULT_DB_ALIAS,
                              apps=global_apps,
                              **kwargs):
    """
    Copied from django create permissions for model, edited to create view and print for reports
    Reports permissions are created in the database similar to django's

    app.print_clientbalances (ClientBalances is the report slug (defaulting to report class name), Content Type is the
    base model content_type

    :param app_config:
    :param verbosity:
    :param interactive:
    :param using:
    :param apps:
    :param kwargs:
    :return:
    """

    if not app_config.models_module:
        return

    # Ensure that contenttypes are created for this app. Needed if
    # 'django.contrib.auth' is in INSTALLED_APPS before
    # 'django.contrib.contenttypes'.
    create_contenttypes(app_config,
                        verbosity=verbosity,
                        interactive=interactive,
                        using=using,
                        apps=apps,
                        **kwargs)

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    all_reports = report_registry.get_all_reports()
    searched_perms = []
    # The codenames and ctypes that should exist.
    ctypes = set()
    for report in all_reports:
        # for klass in report_registry.get_base_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        klass = report.base_model
        ctype = ContentType.objects.db_manager(using).get_for_model(
            klass, for_concrete_model=False)

        ctypes.add(ctype)
        for perm in _get_all_permissions(report):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("content_type", "codename"))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 34
0
def update_contenttypes(app_config,
                        verbosity=2,
                        interactive=True,
                        using=DEFAULT_DB_ALIAS,
                        apps=global_apps,
                        **kwargs):
    """
    Creates content types for models in the given app, removing any model
    entries that no longer have a matching model class.
    """
    if not app_config.models_module:
        return

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        return

    if not router.allow_migrate_model(using, ContentType):
        return

    ContentType.objects.clear_cache()
    # Always clear the global content types cache.
    if apps is not global_apps:
        global_apps.get_model('contenttypes',
                              'ContentType').objects.clear_cache()

    app_models = {
        model._meta.model_name: model
        for model in app_config.get_models()
    }

    if not app_models:
        return

    # Get all the content types
    content_types = {
        ct.model: ct
        for ct in ContentType.objects.using(using).filter(app_label=app_label)
    }
    to_remove = [
        ct for (model_name, ct) in six.iteritems(content_types)
        if model_name not in app_models
    ]

    cts = [
        ContentType(
            app_label=app_label,
            model=model_name,
        ) for (model_name, model) in six.iteritems(app_models)
        if model_name not in content_types
    ]
    ContentType.objects.using(using).bulk_create(cts)
    if verbosity >= 2:
        for ct in cts:
            print("Adding content type '%s | %s'" % (ct.app_label, ct.model))

    # Confirm that the content type is stale before deletion.
    using = router.db_for_write(ContentType)
    if to_remove:
        if interactive:
            ct_info = []
            for ct in to_remove:
                ct_info.append('    - Content type for %s.%s' %
                               (ct.app_label, ct.model))
                collector = NoFastDeleteCollector(using=using)
                collector.collect([ct])

                for obj_type, objs in collector.data.items():
                    if objs == {ct}:
                        continue
                    ct_info.append('    - %s %s object(s)' % (
                        len(objs),
                        obj_type._meta.label,
                    ))

            content_type_display = '\n'.join(ct_info)
            print(
                """Some content types in your database are stale and can be deleted.
Any objects that depend on these content types will also be deleted.
The content types and dependent objects that would be deleted are:

%s

This list doesn't include any cascade deletions to data outside of Django's
models (uncommon).

Are you sure you want to delete these content types?
If you're unsure, answer 'no'.
    """ % content_type_display)
            ok_to_delete = input("Type 'yes' to continue, or 'no' to cancel: ")
        else:
            ok_to_delete = False

        if ok_to_delete == 'yes':
            for ct in to_remove:
                if verbosity >= 2:
                    print("Deleting stale content type '%s | %s'" %
                          (ct.app_label, ct.model))
                ct.delete()
        else:
            if verbosity >= 2:
                print("Stale content types remain.")
Ejemplo n.º 35
0
def create_permissions(app_config,
                       verbosity=2,
                       interactive=True,
                       using=DEFAULT_DB_ALIAS,
                       apps=global_apps,
                       **kwargs):
    if not app_config.models_module:
        return

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = []
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(klass)

        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.

    # For djongo, replace this

    # all_perms = set(Permission.objects.using(using).filter(
    #    content_type__in=ctypes,
    # ).values_list(
    #    "content_type", "codename"
    # ))

    # With this

    all_perms = set()
    ctypes_pks = set(ct.pk for ct in ctypes)
    for ctype, codename in Permission.objects.all().values_list(
            'content_type', 'codename')[:1000000]:
        if ctype in ctypes_pks:
            all_perms.add((ctype, codename))

    # End of the djongo edit

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 36
0
def create_view_permissions(app_config,
                            verbosity=2,
                            interactive=True,
                            using=DEFAULT_DB_ALIAS,
                            **kwargs):  # noqa
    """
    Create 'view' permissions for all models.

    ``django.contrib.auth`` only creates add, change and delete permissions.
    Since we want to support read-only views, we need to add our own
    permission.

    Copied from ``https://github.com/django/django/blob/1.9.6/django/contrib/auth/management/__init__.py#L60``.

    """
    if not app_config.models_module:
        return

    try:
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    from django.contrib.contenttypes.models import ContentType

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(klass)

        ctypes.add(ctype)
        perm = (get_permission_codename('view', klass._meta),
                'Can view %s' % (klass._meta.verbose_name_raw))
        searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("content_type", "codename"))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    # Validate the permissions before bulk_creation to avoid cryptic
    # database error when the verbose_name is longer than 50 characters
    permission_name_max_length = Permission._meta.get_field('name').max_length
    verbose_name_max_length = permission_name_max_length - 11  # len('Can change ') prefix
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            raise ValidationError(
                "The verbose_name of %s.%s is longer than %s characters" % (
                    perm.content_type.app_label,
                    perm.content_type.model,
                    verbose_name_max_length,
                ))
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 37
0
def create_extra_permissions(app_config, verbosity=2, interactive=True,
                             using=DEFAULT_DB_ALIAS, **kwargs):
    """
    Create the extra user & group permissions we need.  Pulled almost entirely from:

    django.contrib.auth.management:create_permissions
    """

    if not app_config.models_module:
        return

    try:
        Permission = apps.get_model('auth', 'Permission')
        User = apps.get_model('auth', 'User')
        Group = apps.get_model('auth', 'Group')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    from django.contrib.contenttypes.models import ContentType

    # The list of perms we need to create
    perms = []

    for model, extra_perms in ((User, EXTRA_USER_PERMS), (Group, EXTRA_GROUP_PERMS)):
        # This will hold the permissions we're looking for as
        # (content_type, (codename, name))
        searched_perms = []

        # Grab the ctype for the model
        ctype = ContentType.objects.db_manager(using).get_for_model(model)

        model_name = model._meta.model_name

        for perm in extra_perms:
            searched_perms.append((ctype, ('%s_%s' % (perm, model_name),
                                           'Can %s %s' % (perm, model_name))))

        # Find all the Permissions that have a content_type for the model.
        # We don't need to check for codenames since we already have
        # a list of the ones we're going to create.
        all_perms = set(Permission.objects.using(using).filter(
            content_type=ctype,
        ).values_list(
            "content_type", "codename"
        ))

        perms += [
            Permission(codename=codename, name=name, content_type=ct)
            for ct, (codename, name) in searched_perms
            if (ct.pk, codename) not in all_perms
        ]

    # Validate the permissions before bulk_creation to avoid cryptic
    # database error when the verbose_name is longer than 50 characters
    permission_name_max_length = Permission._meta.get_field('name').max_length
    verbose_name_max_length = permission_name_max_length - 11  # len('Can change ') prefix
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            raise exceptions.ValidationError(
                "The verbose_name of %s.%s is longer than %s characters" % (
                    perm.content_type.app_label,
                    perm.content_type.model,
                    verbose_name_max_length,
                )
            )

    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 38
0
from collections import OrderedDict
Ejemplo n.º 39
0
def get_contenttypes_and_models(app_config, using, ContentType):
    if not router.allow_migrate_model(using, ContentType):
        return None, None
Ejemplo n.º 40
0
    def handle(self, **options):
        db = options['database']
        include_stale_apps = options['include_stale_apps']
        interactive = options['interactive']
        verbosity = options['verbosity']

        if not router.allow_migrate_model(db, ContentType):
            return
        ContentType.objects.clear_cache()

        apps_content_types = itertools.groupby(
            ContentType.objects.using(db).order_by('app_label', 'model'),
            lambda obj: obj.app_label,
        )
        for app_label, content_types in apps_content_types:
            if not include_stale_apps and app_label not in apps.app_configs:
                continue
            to_remove = [
                ct for ct in content_types if ct.model_class() is None
            ]
            # Confirm that the content type is stale before deletion.
            using = router.db_for_write(ContentType)
            if to_remove:
                if interactive:
                    ct_info = []
                    for ct in to_remove:
                        ct_info.append('    - Content type for %s.%s' %
                                       (ct.app_label, ct.model))
                        collector = NoFastDeleteCollector(using=using)
                        collector.collect([ct])

                        for obj_type, objs in collector.data.items():
                            if objs != {ct}:
                                ct_info.append('    - %s %s object(s)' % (
                                    len(objs),
                                    obj_type._meta.label,
                                ))
                    content_type_display = '\n'.join(ct_info)
                    self.stdout.write(
                        """Some content types in your database are stale and can be deleted.
Any objects that depend on these content types will also be deleted.
The content types and dependent objects that would be deleted are:

%s

This list doesn't include any cascade deletions to data outside of Django's
models (uncommon).

Are you sure you want to delete these content types?
If you're unsure, answer 'no'.""" % content_type_display)
                    ok_to_delete = input(
                        "Type 'yes' to continue, or 'no' to cancel: ")
                else:
                    ok_to_delete = 'yes'

                if ok_to_delete == 'yes':
                    for ct in to_remove:
                        if verbosity >= 2:
                            self.stdout.write(
                                "Deleting stale content type '%s | %s'" %
                                (ct.app_label, ct.model))
                        ct.delete()
                else:
                    if verbosity >= 2:
                        self.stdout.write("Stale content types remain.")
Ejemplo n.º 41
0
def update_contenttypes(app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, **kwargs):
    """
    Creates content types for models in the given app, removing any model
    entries that no longer have a matching model class.
    """
    if not app_config.models_module:
        return

    try:
        ContentType = apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        return

    if not router.allow_migrate_model(using, ContentType):
        return

    ContentType.objects.clear_cache()

    app_label = app_config.label

    app_models = {
        model._meta.model_name: model
        for model in app_config.get_models()}

    if not app_models:
        return

    # Get all the content types
    content_types = {
        ct.model: ct
        for ct in ContentType.objects.using(using).filter(app_label=app_label)
    }
    to_remove = [
        ct
        for (model_name, ct) in six.iteritems(content_types)
        if model_name not in app_models
    ]

    cts = [
        ContentType(
            app_label=app_label,
            model=model_name,
        )
        for (model_name, model) in six.iteritems(app_models)
        if model_name not in content_types
    ]
    ContentType.objects.using(using).bulk_create(cts)
    if verbosity >= 2:
        for ct in cts:
            print("Adding content type '%s | %s'" % (ct.app_label, ct.model))

    # Confirm that the content type is stale before deletion.
    if to_remove:
        if interactive:
            content_type_display = '\n'.join(
                '    %s | %s' % (ct.app_label, ct.model)
                for ct in to_remove
            )
            ok_to_delete = input("""The following content types are stale and need to be deleted:

%s

Any objects related to these content types by a foreign key will also
be deleted. Are you sure you want to delete these content types?
If you're unsure, answer 'no'.

    Type 'yes' to continue, or 'no' to cancel: """ % content_type_display)
        else:
            ok_to_delete = False

        if ok_to_delete == 'yes':
            for ct in to_remove:
                if verbosity >= 2:
                    print("Deleting stale content type '%s | %s'" % (ct.app_label, ct.model))
                ct.delete()
        else:
            if verbosity >= 2:
                print("Stale content types remain.")
Ejemplo n.º 42
0
        def get_objects(count_only=False):
            """
            Collate the objects to be serialized. If count_only is True, just
            count the number of objects to be serialized.
            """
            models = serializers.sort_dependencies(app_list.items())
            for model in models:
                if model in excluded_models:
                    continue
                if model._meta.proxy and model._meta.proxy_for_model not in models:
                    warnings.warn(
                        "%s is a proxy model and won't be serialized." % model._meta.label,
                        category=ProxyModelWarning,
                    )
                if not model._meta.proxy and router.allow_migrate_model(using, model):
                    if use_base_manager:
                        objects = model._base_manager
                    else:
                        objects = model._default_manager

                    queryset = objects.using(using).order_by(model._meta.pk.name)
                    if primary_keys:
                        queryset = queryset.filter(pk__in=primary_keys)
                    if count_only:
                        yield queryset.order_by().count()
                    else:
<<<<<<< HEAD
                        for obj in queryset.iterator():
                            yield obj
=======
Ejemplo n.º 43
0
def create_proxy_permissions(
        app_config, verbosity=2, interactive=True,
        using=DEFAULT_DB_ALIAS, **kwargs):
    if not app_config.models_module:
        return
    logger = logging.getLogger(__name__)
    # print(app_config)

    try:
        logger.info("Tentando obter modelo de permissão do app.")
        Permission = django.apps.apps.get_model('auth', 'Permission')
    except LookupError as e:
        logger.error(str(e))
        return

    if not router.allow_migrate_model(using, Permission):
        return

    from django.contrib.contenttypes.models import ContentType

    permission_name_max_length = Permission._meta.get_field('name').max_length

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in list(app_config.get_models()):
        opts = klass._meta
        permissions = (
            ("list_" + opts.model_name,
             string_concat(
                 _('Visualizaçao da lista de'), ' ',
                 opts.verbose_name_plural)),
            ("detail_" + opts.model_name,
             string_concat(
                 _('Visualização dos detalhes de'), ' ',
                 opts.verbose_name_plural)),
        )
        opts.permissions = tuple(
            set(list(permissions) + list(opts.permissions)))

        if opts.proxy:
            # Force looking up the content types in the current database
            # before creating foreign keys to them.
            app_label, model = opts.app_label, opts.model_name

            try:
                logger.info("Tentando obter db_manager.")
                ctype = ContentType.objects.db_manager(
                    using).get_by_natural_key(app_label, model)
            except Exception as e:
                logger.error(str(e))
                ctype = ContentType.objects.db_manager(
                    using).create(app_label=app_label, model=model)
        else:
            ctype = ContentType.objects.db_manager(using).get_for_model(klass)

        ctypes.add(ctype)

        # FIXME: Retirar try except quando sapl passar a usar django 1.11
        try:
            logger.info("_get_all_permissions")
            # Função não existe mais em Django 1.11
            # como sapl ainda não foi para Django 1.11
            # esta excessão foi adicionada para caso o
            # Sapl esteja rodando em um projeto 1.11 não ocorra erros
            _all_perms_of_klass = _get_all_permissions(klass._meta, ctype)
        except Exception as e:
            logger.error(str(e))
            # Nova função usada em projetos com Django 1.11 e o sapl é uma app
            _all_perms_of_klass = _get_all_permissions(klass._meta)

        for perm in _all_perms_of_klass:
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(Permission.objects.using(using).filter(
        content_type__in=ctypes,
    ).values_list(
        "content_type", "codename"
    ))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    # Validate the permissions before bulk_creation to avoid cryptic database
    # error when the name is longer than 255 characters
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            logger.error("The permission name %s of %s.%s "
                         "is longer than %s characters" % (
                             perm.name,
                             perm.content_type.app_label,
                             perm.content_type.model,
                             permission_name_max_length,
                         ))
            raise exceptions.ValidationError(
                'The permission name %s of %s.%s '
                'is longer than %s characters' % (
                    perm.name,
                    perm.content_type.app_label,
                    perm.content_type.model,
                    permission_name_max_length,
                )
            )
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 44
0
    def handle(self, *app_labels, **options):
        format = options.get('format')
        indent = options.get('indent')
        using = options.get('database')
        excludes = options.get('exclude')
        output = options.get('output')
        output_folder = options.get('output_folder')
        print "Output folder:", output_folder
        print "NOTE: See --output-folder option"
        max_records_per_chunk = options.get('max_records_per_chunk')
        show_traceback = options.get('traceback')
        use_natural_foreign_keys = options.get('use_natural_foreign_keys')
        use_natural_primary_keys = options.get('use_natural_primary_keys')
        use_base_manager = options.get('use_base_manager')
        pks = options.get('primary_keys')

        if pks:
            primary_keys = pks.split(',')
        else:
            primary_keys = []

        excluded_apps = set()
        excluded_models = set()
        for exclude in excludes:
            if '.' in exclude:
                try:
                    model = apps.get_model(exclude)
                except LookupError:
                    raise CommandError('Unknown model in excludes: %s' %
                                       exclude)
                excluded_models.add(model)
            else:
                try:
                    app_config = apps.get_app_config(exclude)
                except LookupError as e:
                    raise CommandError(str(e))
                excluded_apps.add(app_config)

        if len(app_labels) == 0:
            if primary_keys:
                raise CommandError(
                    "You can only use --pks option with one model")
            app_list = OrderedDict((app_config, None)
                                   for app_config in apps.get_app_configs()
                                   if app_config.models_module is not None
                                   and app_config not in excluded_apps)
        else:
            if len(app_labels) > 1 and primary_keys:
                raise CommandError(
                    "You can only use --pks option with one model")
            app_list = OrderedDict()
            for label in app_labels:
                try:
                    app_label, model_label = label.split('.')
                    try:
                        app_config = apps.get_app_config(app_label)
                    except LookupError as e:
                        raise CommandError(str(e))
                    if app_config.models_module is None or app_config in excluded_apps:
                        continue
                    try:
                        model = app_config.get_model(model_label)
                    except LookupError:
                        raise CommandError("Unknown model: %s.%s" %
                                           (app_label, model_label))

                    app_list_value = app_list.setdefault(app_config, [])

                    # We may have previously seen a "all-models" request for
                    # this app (no model qualifier was given). In this case
                    # there is no need adding specific models to the list.
                    if app_list_value is not None:
                        if model not in app_list_value:
                            app_list_value.append(model)
                except ValueError:
                    if primary_keys:
                        raise CommandError(
                            "You can only use --pks option with one model")
                    # This is just an app - no model qualifier
                    app_label = label
                    try:
                        app_config = apps.get_app_config(app_label)
                    except LookupError as e:
                        raise CommandError(str(e))
                    if app_config.models_module is None or app_config in excluded_apps:
                        continue
                    app_list[app_config] = None

        # Check that the serialization format exists; this is a shortcut to
        # avoid collating all the objects and _then_ failing.
        if format not in serializers.get_public_serializer_formats():
            try:
                serializers.get_serializer(format)
            except serializers.SerializerDoesNotExist:
                pass

            raise CommandError("Unknown serialization format: %s" % format)

        # def get_objects(count_only=False):
        #     """
        #     Collate the objects to be serialized. If count_only is True, just
        #     count the number of objects to be serialized.
        #     """
        #     for model in serializers.sort_dependencies(app_list.items()):
        #         if model in excluded_models:
        #             continue
        #         if not model._meta.proxy and router.allow_migrate_model(using, model):
        #             if use_base_manager:
        #                 objects = model._base_manager
        #             else:
        #                 objects = model._default_manager
        #
        #             queryset = objects.using(using).order_by(model._meta.pk.name)
        #             if primary_keys:
        #                 queryset = queryset.filter(pk__in=primary_keys)
        #             if count_only:
        #                 yield queryset.order_by().count()
        #             else:
        #                 for obj in queryset.iterator():
        #                     yield obj
        #
        # try:
        #     self.stdout.ending = None
        #     progress_output = None
        #     object_count = 0
        #     # If dumpdata is outputting to stdout, there is no way to display progress
        #     if (output and self.stdout.isatty() and options['verbosity'] > 0):
        #         progress_output = self.stdout
        #         object_count = sum(get_objects(count_only=True))
        #     stream = open(output, 'w') if output else None
        #     try:
        #         chunk_count = 1000
        #         serializers.serialize(format, get_objects(), indent=indent,
        #                 use_natural_foreign_keys=use_natural_foreign_keys,
        #                 use_natural_primary_keys=use_natural_primary_keys,
        #                 stream=stream or self.stdout, progress_output=progress_output,
        #                 object_count=object_count)
        #     finally:
        #         if stream:
        #             stream.close()
        # except Exception as e:
        #     if show_traceback:
        #         raise
        #     raise CommandError("Unable to serialize database: %s" % e)

        objects = []
        model_count = 1000
        chunk_count = 1000
        for model in serializers.sort_dependencies(app_list.items()):
            model_count += 1
            if model in excluded_models:
                continue
            if not model._meta.proxy and router.allow_migrate_model(
                    using, model):
                if use_base_manager:
                    objects.extend(model._base_manager.using(using).all())
                else:
                    items_total = model._default_manager.using(using).count()
                    chunks_total = (items_total / max_records_per_chunk) + 1
                    for chunk_num in range(0, chunks_total):
                        output_objects = model._default_manager.using(
                            using).all().order_by(
                                model._meta.pk.name)[chunk_num *
                                                     max_records_per_chunk:
                                                     (chunk_num + 1) *
                                                     max_records_per_chunk]
                        if output_objects:
                            chunk_count += 1
                            dump_file_name = output_folder + "/%d_%d.json" % (
                                model_count, chunk_count)
                            print "Dumping file: %s [%d]" % (dump_file_name,
                                                             chunks_total)
                            output = serializers.serialize(
                                format,
                                output_objects,
                                indent=indent,
                                use_natural_foreign_keys=
                                use_natural_foreign_keys,
                                use_natural_primary_keys=
                                use_natural_primary_keys)
                            with open(dump_file_name, "w") as dumpfile:
                                dumpfile.write(output)
Ejemplo n.º 45
0
def update_contenttypes(app_config,
                        verbosity=2,
                        interactive=True,
                        using=DEFAULT_DB_ALIAS,
                        **kwargs):
    """
    Creates content types for models in the given app, removing any model
    entries that no longer have a matching model class.
    """
    if not app_config.models_module:
        return

    try:
        ContentType = apps.get_model('contenttypes', 'ContentType')
    except LookupError:
        return

    if not router.allow_migrate_model(using, ContentType):
        return

    ContentType.objects.clear_cache()

    app_label = app_config.label

    app_models = {
        model._meta.model_name: model
        for model in app_config.get_models()
    }

    if not app_models:
        return

    # Get all the content types
    content_types = {
        ct.model: ct
        for ct in ContentType.objects.using(using).filter(app_label=app_label)
    }
    to_remove = [
        ct for (model_name, ct) in six.iteritems(content_types)
        if model_name not in app_models
    ]

    cts = [
        ContentType(
            app_label=app_label,
            model=model_name,
        ) for (model_name, model) in six.iteritems(app_models)
        if model_name not in content_types
    ]
    ContentType.objects.using(using).bulk_create(cts)
    if verbosity >= 2:
        for ct in cts:
            print("Adding content type '%s | %s'" % (ct.app_label, ct.model))

    # Confirm that the content type is stale before deletion.
    if to_remove:
        if interactive:
            content_type_display = '\n'.join(
                '    %s | %s' % (ct.app_label, ct.model) for ct in to_remove)
            ok_to_delete = input(
                """The following content types are stale and need to be deleted:

%s

Any objects related to these content types by a foreign key will also
be deleted. Are you sure you want to delete these content types?
If you're unsure, answer 'no'.

    Type 'yes' to continue, or 'no' to cancel: """ % content_type_display)
        else:
            ok_to_delete = False

        if ok_to_delete == 'yes':
            for ct in to_remove:
                if verbosity >= 2:
                    print("Deleting stale content type '%s | %s'" %
                          (ct.app_label, ct.model))
                ct.delete()
        else:
            if verbosity >= 2:
                print("Stale content types remain.")
Ejemplo n.º 46
0
Archivo: apps.py Proyecto: alroleg/sapl
def create_proxy_permissions(app_config,
                             verbosity=2,
                             interactive=True,
                             using=DEFAULT_DB_ALIAS,
                             **kwargs):
    if not app_config.models_module:
        return

    # print(app_config)

    try:
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    from django.contrib.contenttypes.models import ContentType

    permission_name_max_length = Permission._meta.get_field('name').max_length

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in list(app_config.get_models()):
        opts = klass._meta
        permissions = (
            ("list_" + opts.model_name,
             string_concat(_('Visualizaçao da lista de'), ' ',
                           opts.verbose_name_plural)),
            ("detail_" + opts.model_name,
             string_concat(_('Visualização dos detalhes de'), ' ',
                           opts.verbose_name_plural)),
        )
        opts.permissions = tuple(
            set(list(permissions) + list(opts.permissions)))

        if opts.proxy:
            # Force looking up the content types in the current database
            # before creating foreign keys to them.
            app_label, model = opts.app_label, opts.model_name

            try:
                ctype = ContentType.objects.db_manager(
                    using).get_by_natural_key(app_label, model)
            except:
                ctype = ContentType.objects.db_manager(using).create(
                    app_label=app_label, model=model)
        else:
            ctype = ContentType.objects.db_manager(using).get_for_model(klass)

        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta, ctype):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("content_type", "codename"))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    # Validate the permissions before bulk_creation to avoid cryptic database
    # error when the name is longer than 255 characters
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            raise exceptions.ValidationError('The permission name %s of %s.%s '
                                             'is longer than %s characters' % (
                                                 perm.name,
                                                 perm.content_type.app_label,
                                                 perm.content_type.model,
                                                 permission_name_max_length,
                                             ))
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
    def create_table(self, database, tablename, dry_run):
        cache = BaseDatabaseCache(tablename, {})
        if not router.allow_migrate_model(database, cache.cache_model_class):
            return
        connection = connections[database]

        if tablename in connection.introspection.table_names():
            if self.verbosity > 0:
                self.stdout.write("Cache table '%s' already exists." % tablename)
            return

        fields = (
            # "key" is a reserved word in MySQL, so use "cache_key" instead.
            models.CharField(
                name="cache_key", max_length=255, unique=True, primary_key=True
            ),
            models.TextField(name="value"),
            models.DateTimeField(name="expires", db_index=True),
        )
        table_output = []
        index_output = []
        qn = connection.ops.quote_name
        for f in fields:
            field_output = [
                qn(f.name),
                f.db_type(connection=connection),
                "%sNULL" % ("NOT " if not f.null else ""),
            ]
            if f.primary_key:
                field_output.append("PRIMARY KEY")
            elif f.unique:
                field_output.append("UNIQUE")
            if f.db_index:
                unique = "UNIQUE " if f.unique else ""
                index_output.append(
                    "CREATE %sINDEX %s ON %s (%s);"
                    % (
                        unique,
                        qn("%s_%s" % (tablename, f.name)),
                        qn(tablename),
                        qn(f.name),
                    )
                )
            table_output.append(" ".join(field_output))
        full_statement = ["CREATE TABLE %s (" % qn(tablename)]
        for i, line in enumerate(table_output):
            full_statement.append(
                "    %s%s" % (line, "," if i < len(table_output) - 1 else "")
            )
        full_statement.append(");")

        full_statement = "\n".join(full_statement)

        if dry_run:
            self.stdout.write(full_statement)
            for statement in index_output:
                self.stdout.write(statement)
            return

        with transaction.atomic(
            using=database, savepoint=connection.features.can_rollback_ddl
        ):
            with connection.cursor() as curs:
                try:
                    curs.execute(full_statement)
                except DatabaseError as e:
                    raise CommandError(
                        "Cache table '%s' could not be created.\nThe error was: %s."
                        % (tablename, e)
                    )
                for statement in index_output:
                    curs.execute(statement)

        if self.verbosity > 1:
            self.stdout.write("Cache table '%s' created." % tablename)
Ejemplo n.º 48
0
def create_permissions(  # NOQA
    app_config,
    verbosity=2,
    interactive=True,
    using=DEFAULT_DB_ALIAS,
    apps=global_apps,
    **kwargs
):
    if not app_config.models_module:
        return

    from field_permissions.registry import field_permissions

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model("contenttypes", "ContentType")  # NOQA
        Permission = apps.get_model("auth", "Permission")  # NOQA
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = []
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        if not field_permissions.in_registry(klass):
            continue

        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(klass)

        ctypes.add(ctype)
        for perm in field_permissions.get_field_permissions_for_model(klass):
            searched_perms.append((ctype, perm))

    if not searched_perms:
        return

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using)
        .filter(content_type__in=ctypes)
        .values_list("content_type", "codename")
    )

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]

    Permission.objects.using(using).bulk_create(perms)

    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '{}'".format(perm.codename))  # NOQA
Ejemplo n.º 49
0
    def load_label(self, fixture_label):
        """
        Loads fixtures files for a given label.
        """
        show_progress = self.verbosity >= 3
        for fixture_file, fixture_dir, fixture_name in self.find_fixtures(
                fixture_label):
            _, ser_fmt, cmp_fmt = self.parse_name(
                os.path.basename(fixture_file))
            open_method, mode = self.compression_formats[cmp_fmt]
            fixture = open_method(fixture_file, mode)
            try:
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write(
                        "Installing %s fixture '%s' from %s." %
                        (ser_fmt, fixture_name, humanize(fixture_dir)))

                objects = serializers.deserialize(
                    ser_fmt,
                    fixture,
                    using=self.using,
                    ignorenonexistent=self.ignore)

                for obj in objects:
                    objects_in_fixture += 1
                    if router.allow_migrate_model(self.using,
                                                  obj.object.__class__):
                        loaded_objects_in_fixture += 1
                        self.models.add(obj.object.__class__)
                        try:
                            obj.save(using=self.using)
                            if show_progress:
                                self.stdout.write('\rProcessed %i object(s).' %
                                                  loaded_objects_in_fixture,
                                                  ending='')
                        except (DatabaseError, IntegrityError) as e:
                            e.args = (
                                "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s"
                                % {
                                    'app_label': obj.object._meta.app_label,
                                    'object_name':
                                    obj.object._meta.object_name,
                                    'pk': obj.object.pk,
                                    'error_msg': force_text(e)
                                }, )
                            raise
                if objects and show_progress:
                    self.stdout.write(
                        '')  # add a newline after progress indicator
                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                if not isinstance(e, CommandError):
                    e.args = ("Problem installing fixture '%s': %s" %
                              (fixture_file, e), )
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture_name, RuntimeWarning)
Ejemplo n.º 50
0
import sys
Ejemplo n.º 51
0
def create_permissions(app_config,
                       verbosity=2,
                       interactive=True,
                       using=DEFAULT_DB_ALIAS,
                       apps=global_apps,
                       **kwargs):
    if not app_config.models_module:
        return

    # Ensure that contenttypes are created for this BLOG. Needed if
    # 'django.contrib.auth' is in INSTALLED_APPS before
    # 'django.contrib.contenttypes'.
    create_contenttypes(app_config,
                        verbosity=verbosity,
                        interactive=interactive,
                        using=using,
                        apps=apps,
                        **kwargs)

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = []
    # The codenames and ctypes that should exist.
    ctypes = set()
    for klass in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(
            klass, for_concrete_model=False)

        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("content_type", "codename"))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 52
0
    def create(self, request):
        objects_in_fixture = 0
        loaded_objects_count = 0
        models = set()

        fixture = request.data.get('fixture')

        if fixture:
            with transaction.atomic():
                connection = connections[DEFAULT_DB_ALIAS]

                with connection.constraint_checks_disabled():
                    objects = Deserializer(fixture,
                                           ignorenonexistent=True,
                                           handle_forward_references=True)
                    objs_with_deferred_fields = []

                    for obj in objects:
                        objects_in_fixture += 1
                        if router.allow_migrate_model(DEFAULT_DB_ALIAS,
                                                      obj.object.__class__):
                            loaded_objects_count += 1
                            models.add(obj.object.__class__)
                            try:
                                obj.save()
                            # psycopg2 raises ValueError if data contains NUL chars.
                            except (DatabaseError, IntegrityError,
                                    ValueError) as e:
                                e.args = (
                                    "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s"
                                    % {
                                        'app_label':
                                        obj.object._meta.app_label,
                                        'object_name':
                                        obj.object._meta.object_name,
                                        'pk': obj.object.pk,
                                        'error_msg': e,
                                    }, )
                                raise
                        if obj.deferred_fields:
                            objs_with_deferred_fields.append(obj)

                    for obj in objs_with_deferred_fields:
                        obj.save_deferred_fields()

                if loaded_objects_count > 0:
                    sequence_sql = connection.ops.sequence_reset_sql(
                        no_style(), models)
                    if sequence_sql:
                        with connection.cursor() as cursor:
                            for line in sequence_sql:
                                cursor.execute(line)
                # Close the DB connection -- unless we're still in a transaction. This
                # is required as a workaround for an edge case in MySQL: if the same
                # connection is used to create tables, load data, and query, the query
                # can return incorrect results. See Django #7572, MySQL #37735.
            if transaction.get_autocommit():
                connections[DEFAULT_DB_ALIAS].close()

        return Response({"objects_loaded": loaded_objects_count},
                        status=status.HTTP_200_OK)
Ejemplo n.º 53
0
def create_permissions(app_config,
                       verbosity=2,
                       interactive=True,
                       using=DEFAULT_DB_ALIAS,
                       apps=global_apps,
                       **kwargs):

    # run auth's create_permissions since we are re-writing the signal handler
    auth_perms(app_config, verbosity, interactive, using, apps, **kwargs)

    if not app_config.models_module:
        return

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
        Group = apps.get_model('auth', 'Group')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # The ctypes we will review
    ctypes = set()
    for model in app_config.get_models():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(model)
        ctypes.add(ctype)

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("codename", flat=True))

    def post_process_urlpatterns(urlpatterns, base='', namespace=None):

        patterns, resolvers = {}, {}
        for p in urlpatterns:
            if isinstance(p, RegexURLPattern):
                try:
                    if not p.name:
                        name = p.name
                    elif namespace:
                        name = '{0}:{1}'.format(namespace, p.name)
                    else:
                        name = p.name
                    if name in patterns:
                        raise ValueError('"{}" is duplicate')

                    # we are presuming only CBV... could handle FBV one day
                    view_class = p.callback.view_class if hasattr(
                        p.callback, 'view_class') else None
                    view_model = None
                    if view_class:
                        view_initkwargs = p.callback.view_initkwargs
                        view_controller = p.callback.view_initkwargs.get(
                            'controller')
                        view_model = view_controller.model if view_controller else None

                    if view_model:
                        content_type = ContentType.objects.get_for_model(
                            view_model)
                        if content_type not in ctypes:
                            continue
                        view_mode = view_class.mode or view_initkwargs['mode']
                        verbose_name_plural = view_model._meta.verbose_name_plural
                    else:
                        # we cannot add perms on models not in the calling model's app
                        # we will not add perms for non-model views because Perms do not allow
                        # we will always create and refresh groups
                        continue
                        content_type = None
                        name_parts = name.split(':')
                        view_mode = name_parts[-1]
                        verbose_name_plural = '{}'.format(':'.join(
                            name_parts[:-1]))
                    permission_name = 'Can {} {}'.format(
                        view_mode, verbose_name_plural)
                    permission = Permission(codename=name,
                                            name=permission_name,
                                            content_type=content_type)
                    patterns[name] = (base + p.regex.pattern, p.callback,
                                      permission)
                except ViewDoesNotExist:
                    continue
            elif isinstance(p, RegexURLResolver):
                try:
                    url_patterns = p.url_patterns
                except ImportError:
                    continue
                if namespace and p.namespace:
                    _namespace = '{0}:{1}'.format(namespace, p.namespace)
                else:
                    _namespace = (p.namespace or namespace)
                if isinstance(p, LocaleRegexURLResolver):
                    for langauge in settings.LANGUAGES:
                        with translation.override(langauge[0]):
                            if _namespace in resolvers:
                                raise ValueError('"{}" is duplicate')
                            resolvers[_namespace] = post_process_urlpatterns(
                                url_patterns,
                                base + p.regex.pattern,
                                namespace=_namespace)
                else:
                    if _namespace in resolvers:
                        raise ValueError('"{}" is duplicate')
                    resolvers[_namespace] = post_process_urlpatterns(
                        url_patterns,
                        base + p.regex.pattern,
                        namespace=_namespace)

                # bulk create the permissions
                all_patterns = resolvers[_namespace]['patterns']
                new_patterns, new_permissions = [], []
                for name, data in all_patterns.items():
                    if name not in all_perms:
                        new_patterns.append(name)
                        new_permissions.append(data[2])
                        print("Adding permission '%s'" % name)
                Permission.objects.using(using).bulk_create(new_permissions)

                # get-or-create and update the group
                group, created = Group.objects.using(using).get_or_create(
                    name=_namespace)
                if created:
                    print("Adding group '%s'" % _namespace)
                    missing_patterns = new_patterns
                else:
                    missing_patterns = set(new_patterns) - set(
                        group.permissions.values_list('codename', flat=True))
                if missing_patterns:
                    new_group_permissions = Permission.objects.filter(
                        codename__in=missing_patterns)
                    group.permissions.add(*new_group_permissions)
                    for permission in new_group_permissions:
                        print("Adding permission '%s' to group '%s'" %
                              (permission.codename, _namespace))
                    """
                    Can't do nested groups yet so we will rely on controller traversal.
                    group_groups = Group.objects.filter(name__in=resolvers)
                    group.groups.add(*group_groups)
                    """
        return dict(patterns=patterns, resolvers=resolvers)

    post_process_urlpatterns(get_backend().urls)
Ejemplo n.º 54
0
    def load_label(self, fixture_label):
        """Load fixtures files for a given label."""
        show_progress = self.verbosity >= 3
        for fixture_file, fixture_dir, fixture_name in self.find_fixtures(
                fixture_label):
            _, ser_fmt, cmp_fmt = self.parse_name(
                os.path.basename(fixture_file))
            open_method, mode = self.compression_formats[cmp_fmt]
            fixture = open_method(fixture_file, mode)
            try:
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write(
                        "Installing %s fixture '%s' from %s." %
                        (ser_fmt, fixture_name, humanize(fixture_dir)))

                objects = serializers.deserialize(
                    ser_fmt,
                    fixture,
                    using=self.using,
                    ignorenonexistent=self.ignore,
                    handle_forward_references=True,
                )

                for obj in objects:
                    objects_in_fixture += 1
                    if (obj.object._meta.app_config in self.excluded_apps
                            or type(obj.object) in self.excluded_models):
                        continue
                    if router.allow_migrate_model(self.using,
                                                  obj.object.__class__):
                        loaded_objects_in_fixture += 1
                        self.models.add(obj.object.__class__)
                        try:
                            obj.save(using=self.using)
                            if show_progress:
                                self.stdout.write(
                                    "\rProcessed %i object(s)." %
                                    loaded_objects_in_fixture,
                                    ending="",
                                )
                        # psycopg2 raises ValueError if data contains NUL chars.
                        except (DatabaseError, IntegrityError,
                                ValueError) as e:
                            e.args = (
                                "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s"
                                % {
                                    "app_label": obj.object._meta.app_label,
                                    "object_name":
                                    obj.object._meta.object_name,
                                    "pk": obj.object.pk,
                                    "error_msg": e,
                                }, )
                            raise
                    if obj.deferred_fields:
                        self.objs_with_deferred_fields.append(obj)
                if objects and show_progress:
                    self.stdout.write(
                        "")  # add a newline after progress indicator
                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                if not isinstance(e, CommandError):
                    e.args = ("Problem installing fixture '%s': %s" %
                              (fixture_file, e), )
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture_name,
                    RuntimeWarning,
                )
Ejemplo n.º 55
0
def create_extra_permissions(app_config, verbosity=2, interactive=True,
                             using=DEFAULT_DB_ALIAS, **kwargs):
    """
    Create the extra user & group permissions we need.  Pulled almost entirely from:

    django.contrib.auth.management:create_permissions
    """

    if not app_config.models_module:
        return

    try:
        Permission = apps.get_model('auth', 'Permission')
        User = apps.get_model('auth', 'User')
        Group = apps.get_model('auth', 'Group')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    from django.contrib.contenttypes.models import ContentType

    # The list of perms we need to create
    perms = []

    for model, extra_perms in ((User, EXTRA_USER_PERMS), (Group, EXTRA_GROUP_PERMS)):
        # This will hold the permissions we're looking for as
        # (content_type, (codename, name))
        searched_perms = []

        # Grab the ctype for the model
        ctype = ContentType.objects.db_manager(using).get_for_model(model)

        model_name = model._meta.model_name

        for perm in extra_perms:
            searched_perms.append((ctype, ('%s_%s' % (perm, model_name),
                                           'Can %s %s' % (perm, model_name))))

        # Find all the Permissions that have a content_type for the model.
        # We don't need to check for codenames since we already have
        # a list of the ones we're going to create.
        all_perms = set(Permission.objects.using(using).filter(
            content_type=ctype,
        ).values_list(
            "content_type", "codename"
        ))

        perms += [
            Permission(codename=codename, name=name, content_type=ct)
            for ct, (codename, name) in searched_perms
            if (ct.pk, codename) not in all_perms
        ]

    # Validate the permissions before bulk_creation to avoid cryptic
    # database error when the verbose_name is longer than 50 characters
    permission_name_max_length = Permission._meta.get_field('name').max_length
    verbose_name_max_length = permission_name_max_length - 11  # len('Can change ') prefix
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            raise exceptions.ValidationError(
                "The verbose_name of %s.%s is longer than %s characters" % (
                    perm.content_type.app_label,
                    perm.content_type.model,
                    verbose_name_max_length,
                )
            )

    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 56
0
def create_default_environments(app_config,
                                verbosity=2,
                                using=DEFAULT_DB_ALIAS,
                                apps=global_apps,
                                **kwargs):

    try:
        Environment = apps.get_model('configfactory', 'Environment')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Environment):
        return

    environments = settings.DEFAULT_ENVIRONMENTS
    aliases = set()

    # Create or active environments
    for order, environment_data in enumerate(environments, start=1):

        alias = environment_data['alias']  # type: str
        name = environment_data.get('name', alias.title())
        aliases.add(alias)

        try:
            environment = Environment.objects.using(using).get(alias=alias)
            if verbosity >= 2:
                print(f"Updating {alias} environment.")
            environment.name = name
            environment.order = order
            environment.is_active = True
            environment.save(using=using)
        except Environment.DoesNotExist:
            environment = Environment(alias=alias, name=name, order=order)
            if verbosity >= 2:
                print(f"Creating {alias} environment.")
            environment.save(using=using)

    # Deactivate old environment
    for environment in Environment.objects.using(using).all():
        alias = environment.alias
        aliases.add(alias)
        if alias not in aliases and alias != settings.BASE_ENVIRONMENT:
            if verbosity >= 2:
                print(f"Deactivating {alias} environment.")
            environment.is_active = False
            environment.save(using=using)

    # Set environment fallback
    for environment_data in environments:
        alias = environment_data['alias']
        fallback = environment_data.get('fallback')
        try:
            environment = Environment.objects.using(using).get(alias=alias)
            fallback_environment = Environment.objects.using(using).filter(
                alias=fallback).first()
            if verbosity >= 2:
                print(f"Setting {environment.alias} fallback environment.")
            environment.fallback = fallback_environment
            environment.save(using=using)
        except Environment.DoesNotExist:
            continue

    # Create base environment
    if settings.BASE_ENVIRONMENT not in aliases:

        alias = settings.BASE_ENVIRONMENT
        name = alias.title()

        environment = Environment()
        environment.alias = alias
        environment.name = name
        environment.order = -1
        environment.is_active = True
        environment.save(using=using)
    def load_label(self):
        """
        Loads fixtures files for a given label.
        """
        show_progress = self.verbosity >= 3
        for fixture_file, fixture_dir, fixture_name in self.find_fixtures():
            _, ser_fmt = self.parse_name(os.path.basename(fixture_file))
            fixture = open(fixture_file, 'rb')
            try:
                self.fixture_count += 1
                objects_in_fixture = 0
                loaded_objects_in_fixture = 0
                if self.verbosity >= 2:
                    self.stdout.write(
                        "Installing %s fixture '%s' from %s." %
                        (ser_fmt, fixture_name, humanize(fixture_dir)))

                objects = serializers.deserialize(
                    ser_fmt,
                    fixture,
                    using=self.using,
                    ignorenonexistent=self.ignore)

                for obj in objects:
                    objects_in_fixture += 1
                    if router.allow_migrate_model(self.using,
                                                  obj.object.__class__):
                        loaded_objects_in_fixture += 1
                        self.models.add(obj.object.__class__)
                        try:
                            keys = dict([(key, getattr(obj.object, key))
                                         for key in obj.object.unique_together
                                         ])
                            if self.update:
                                vals = dict([(k, v) for k, v in model_to_dict(
                                    obj.object).items() if v is not None])

                                item, created = obj.object.__class__.objects.get_or_create(
                                    defaults=vals, **keys)
                                if not created:
                                    if show_progress:
                                        self.stdout.write(
                                            'Updated an object(s).', ending='')
                                    for k, v in vals.items():
                                        setattr(item, k, v)
                                    item.save()
                            else:
                                if not obj.object.__class__.objects.filter(
                                        **keys).exists():
                                    obj.object.save()
                            if show_progress:
                                self.stdout.write('\rProcessed %i object(s).' %
                                                  loaded_objects_in_fixture,
                                                  ending='')
                        except (DatabaseError, IntegrityError) as e:
                            e.args = (
                                "Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s"
                                % {
                                    'app_label': obj.object._meta.app_label,
                                    'object_name':
                                    obj.object._meta.object_name,
                                    'pk': obj.object.pk,
                                    'error_msg': force_text(e)
                                }, )
                            raise
                if objects and show_progress:
                    self.stdout.write(
                        '')  # add a newline after progress indicator
                self.loaded_object_count += loaded_objects_in_fixture
                self.fixture_object_count += objects_in_fixture
            except Exception as e:
                if not isinstance(e, CommandError):
                    e.args = ("Problem installing fixture '%s': %s" %
                              (fixture_file, e), )
                raise
            finally:
                fixture.close()

            # Warn if the fixture we loaded contains 0 objects.
            if objects_in_fixture == 0:
                warnings.warn(
                    "No fixture data found for '%s'. (File format may be "
                    "invalid.)" % fixture_name, RuntimeWarning)
Ejemplo n.º 58
0
 def get_objects():
     for model in serializers.sort_dependencies(app_list):
         if (model._meta.can_migrate(self.connection) and
                 router.allow_migrate_model(self.connection.alias, model)):
             queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
             yield from queryset.iterator()
Ejemplo n.º 59
0
def create_permissions(app_config,
                       verbosity=2,
                       interactive=True,
                       using=DEFAULT_DB_ALIAS,
                       apps=global_apps,
                       **kwargs):
    """
    自定义创建权限数据
    """
    if not app_config.models_module:
        return

    app_label = app_config.label
    try:
        app_config = apps.get_app_config(app_label)
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, Permission):
        return

    # 删除django auth创建的权限数据
    Permission.objects.all().delete()

    # 全部的app模型
    app_label_list = map(
        lambda item: item[0],
        set(ContentType.objects.all().values_list('app_label')))

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()
    for app_label in app_label_list:
        app_config = apps.get_app_config(app_label)
        for klass in app_config.get_models():
            # Force looking up the content types in the current database
            # before creating foreign keys to them.
            ctype = ContentType.objects.db_manager(using).get_for_model(klass)

            ctypes.add(ctype)
            for perm in _get_all_permissions(klass._meta):
                searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a model we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("content_type", "codename"))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s'" % perm)
Ejemplo n.º 60
0
def update_ctypes_permissions(app_config,
                              verbosity=2,
                              interactive=True,
                              using=DEFAULT_DB_ALIAS,
                              **kwargs):
    """
    Creates resources content types and default permissions.

    Most bits taken from contenttypes.management.update_contenttypes
    and auth.management.create_permissions.

    This will not make any attempt to clear content types no longer
    associated to resources.

    Note:
    -----
    Because content types will not remain associated with real models,
    Django will ask to remove them after every migration.
    Although this function will recreate them, model instances with
    non-weak relations to them will be also deleted by the cascade,
    so for the time being is better to say "no" asked to remove stale
    content types.

    TODO: This *may* be addressed/mitigated defining a database
    router with a defined allow_migrate() method that checks the app
    against ContentType; e.g.:

    def allow_migrate(self, db, app_label, model_name=None, **hints):
        if <this app_label comes from a restorm app>:
            # don't allow the migration
            # (although maybe it's worth checking mode_name)
            return False
        # no opinion (let other routers decide)
        return None

    see
    docs.djangoproject.com/en/1.8/topics/db/multi-db/#allow_migrate

    """
    if not isinstance(app_config, RestormAppConfig):
        # Any model will end up here, we are only interested in
        # restorm resources.
        return

    if not app_config.models_module:
        # This is left here for compatibility with Django.
        # Works because restorm resources are defined in the models.py.
        return

    try:
        ContentType = apps.get_model('contenttypes', 'ContentType')
        Permission = apps.get_model('auth', 'Permission')
    except LookupError:
        return

    if not router.allow_migrate_model(using, ContentType) \
            or not router.allow_migrate_model(using, Permission):
        return

    ContentType.objects.clear_cache()

    app_label = app_config.label
    app_resources = {
        resource._meta.resource_name: resource
        for resource in app_config.get_resources()
    }

    if not app_resources:
        return

    # Get all the content types for this app
    content_types = {
        ct.model: ct
        for ct in ContentType.objects.using(using).filter(app_label=app_label)
    }

    # Create in memory any missing content type
    cts = [
        ContentType(app_label=app_label, model=resources_name)
        for (resources_name, resource) in six.iteritems(app_resources)
        if resources_name not in content_types
    ]

    # Bulk-create the new instances
    ContentType.objects.using(using).bulk_create(cts)
    if verbosity >= 2:
        msg = "Adding content type '{0.app_label} | {0.model}' (restorm)".format
        for ct in cts:
            print(msg(ct))

    # This will hold the permissions we're looking for as
    # (content_type, (codename, name))
    searched_perms = list()
    # The codenames and ctypes that should exist.
    ctypes = set()

    for klass in app_resources.values():
        # Force looking up the content types in the current database
        # before creating foreign keys to them.
        ctype = ContentType.objects.db_manager(using).get_for_model(klass)
        ctypes.add(ctype)
        for perm in _get_all_permissions(klass._meta, ctype):
            searched_perms.append((ctype, perm))

    # Find all the Permissions that have a content_type for a resource we're
    # looking for.  We don't need to check for codenames since we already have
    # a list of the ones we're going to create.
    all_perms = set(
        Permission.objects.using(using).filter(
            content_type__in=ctypes, ).values_list("content_type", "codename"))

    perms = [
        Permission(codename=codename, name=name, content_type=ct)
        for ct, (codename, name) in searched_perms
        if (ct.pk, codename) not in all_perms
    ]
    # Validate the permissions before bulk_creation to avoid cryptic
    # database error when the verbose_name is longer than 50 characters.
    permission_name_max_length = Permission._meta.get_field('name').max_length
    verbose_name_max_length = permission_name_max_length - 11  # len('Can change ') prefix
    for perm in perms:
        if len(perm.name) > permission_name_max_length:
            raise exceptions.ValidationError(
                "The verbose_name of %s.%s is longer than %s characters" % (
                    perm.content_type.app_label,
                    perm.content_type.model,
                    verbose_name_max_length,
                ))
    Permission.objects.using(using).bulk_create(perms)
    if verbosity >= 2:
        for perm in perms:
            print("Adding permission '%s' (restorm)" % perm)