コード例 #1
0
    def update_topic_stats(self):
        self.stdout.write("\n *** set topic stats...\n")

        topics = Topic.objects.all()
        total = topics.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        start_time = time.time()
        next_status = time.time() + 0.25
        for count, topic in enumerate(topics):
            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                msg = (
                    "\r\t%i/%i topics... rest: %i - eta: %s (rate: %.1f/sec)         "
                ) % (count, total, rest, eta, rate)
                self.stdout.write(msg)
                self.stdout.flush()

            queryset = Post.objects.only("created", "updated").filter(topic=topic)
            topic.post_count = queryset.count()
            try:
                last_post = queryset.latest("updated")
            except Post.DoesNotExist:
                # there is no post in this forum
                pass
            else:
                topic.last_post = last_post
                if last_post.updated:
                    topic.updated = last_post.updated
                else:
                    topic.updated = last_post.created
            topic.save()

        duration = time.time() - start_time
        rate = float(count) / duration
        self.stdout.write(
            "\r *** %i topic stats set in %s (rate: %.1f/sec)\n" % (
                count, human_duration(duration), rate
            )
        )
コード例 #2
0
    def update_topic_stats(self):
        self.out(u"\n *** set topic stats...\n")

        topics = Topic.objects.all()
        total = topics.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        start_time = time.time()
        next_status = time.time() + 0.25
        for count, topic in enumerate(topics, 1):
            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i topics... rest: %i - eta: %s (rate: %.1f/sec)" %
                    (count, total, rest, eta, rate))

            queryset = Post.objects.only("created",
                                         "updated").filter(topic=topic)
            topic.post_count = queryset.count()
            try:
                last_post = queryset.latest("updated")
            except Post.DoesNotExist:
                # there is no post in this forum
                pass
            else:
                topic.last_post = last_post
                if last_post.updated:
                    topic.updated = last_post.updated
                else:
                    topic.updated = last_post.created
            topic.save()

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(" *** %i topic stats set in %s (rate: %.1f/sec)" %
                           (count, human_duration(duration), rate))
コード例 #3
0
class Command(BaseCommand):
    help = 'migrate a phpBB3 installation to DjangoBB'
    option_list = BaseCommand.option_list + (
        make_option(
            '--noinput',
            action='store_false',
            dest='interactive',
            default=True,
            help='Tells Django to NOT prompt the user for input of any kind.'),
        make_option(
            '--cleanup_users',
            action='store',
            dest='cleanup_users',
            default='2',
            type='choice',
            choices=['0', '1', '2', '3'],
            help=
            'Which user to migrate: 0:all users, 1:with email, 2:+lastvisit (default), 3:+has post'
        ),
        make_option(
            '--only_users',
            action='store_true',
            help='Migrate/update only the phpBB users to django users'),
        make_option(
            '--flush_djangobb',
            action='store_true',
            help='Delete all DjangoBB forum models. (For testing, only)'),
        make_option(
            '--max_entries',
            type="int",
            help='Migrate only a few users/topics/posts. (For testing, only)'),
    )

    def handle(self, *args, **options):
        try:
            self._handle(*args, **options)
        except Exception:
            print "-" * TERMINAL_WIDTH
            traceback.print_exc()
            print "-" * TERMINAL_WIDTH
            debugger.post_mortem()

    def _handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity'))
        self.interactive = options.get('interactive')

        only_users = options.get("only_users", False)
        flush_djangobb = options.get("flush_djangobb", False)
        if flush_djangobb:
            self._flush_djangobb()
            return

        self.max_entries = options.get("max_entries")
        if self.max_entries:
            self.warn("Migrate with max entries: %i !\n" % self.max_entries)

        self.check_attachment_path()
        self.check_models()

        # disable DjangoBB signals for speedup
        post_save.disconnect(djangobb_signals.post_saved,
                             sender=Post,
                             dispatch_uid='djangobb_post_save')
        post_save.disconnect(djangobb_signals.topic_saved,
                             sender=Topic,
                             dispatch_uid='djangobb_topic_save')

        # Speedup migration by disable haystack search index creation.
        site.unregister(Post)

        #disable_auto_fields(Forum)
        disable_auto_fields(Topic)
        disable_auto_fields(Post)

        cleanup_users = int(options.get("cleanup_users"))
        moderator_groups = phpbb_Group.objects.filter(
            name__in=["ADMINISTRATORS", "GLOBAL_MODERATORS"])
        user_dict, moderators = self.migrate_users(cleanup_users,
                                                   moderator_groups)
        if only_users:
            self.out(u"\nmigrate only users done.\n")
            return

        forum_dict = self.migrate_forums(moderators)

        self.migrate_topic(user_dict, forum_dict)
        self.migrate_posts(user_dict)

        # needed if signals disabled, see above
        self.update_topic_stats()
        self.update_forum_stats()

        self.out(u"\nmigration done.\n")

    def _out_encode(self, msg):
        # see: https://github.com/jedie/django-phpBB3/issues/9#issuecomment-8494830
        msg = smart_str(msg, encoding=OUT_ENCODING, errors="replace")
        msg = msg.replace("\t", "    ")  # Helpful in out_update()
        return msg

    def warn(self, msg):
        self.stderr.write(self.style.ERROR(self._out_encode(msg)))
        self.stderr.flush()

    def out(self, msg):
        self.stdout.write(self._out_encode(msg))
        self.stdout.flush()

    def err(self, msg):
        self.stderr.write(self._out_encode(msg))
        self.stdout.flush()

    def _out_justify(self, msg):
        self.stdout.write(msg.ljust(TERMINAL_WIDTH))

    def out_update(self, msg):
        msg = self._out_encode(msg)
        self.stdout.write("\r")
        self._out_justify(msg)
        self.stdout.flush()

    def out_overwrite(self, msg):
        self.out_update(msg)
        self.stdout.write("\n")
        self.stdout.flush()

    def _flush_djangobb(self):
        self.warn("Delete all DjangoBB data.\n")
        if self.interactive:
            confirm = raw_input("Continue? (yes/no): ")
            while 1:
                if confirm.lower().startswith("n"):
                    sys.exit("-1")
                if confirm.lower() != "yes":
                    confirm = raw_input('Please enter either "yes" or "no": ')
                    continue
                break

        for ModelClass in (Attachment, Post, Topic, Forum, Profile, Category):
            count = ModelClass.objects.all().count()
            self.out(" *** Delete %i '%s' model entries..." %
                     (count, ModelClass.__name__))
            ModelClass.objects.all().delete()
            self.out("OK\n")

    def check_attachment_path(self):
        attachment_count = phpbb_Attachment.objects.count()
        self.out(
            self.style.NOTICE("This phpBB3 Forum has %i Attachment(s).\n" %
                              attachment_count))
        if attachment_count == 0:
            # Don't check paths if no attachments exists.
            return

        path = getattr(settings, "PHPBB_ATTACHMENT_PATH", None)
        if path is None:
            raise CommandError("settings.PHPBB_ATTACHMENT_PATH was not set!"
                               " Please add it into your local_settings.py!")
        if not os.path.isdir(path):
            msg = ("PHPBB_ATTACHMENT_PATH '%s' doesn't exists!"
                   " Please add it into your local_settings.py!") % path
            raise CommandError(msg)

        for filename in (".htaccess", "index.htm"):
            test_path = os.path.join(path, filename)
            if not os.path.isfile(test_path):
                self.warn("WARNING: file '%s' doesn't exists!\n" % test_path)

        djangobb_path = os.path.join(settings.MEDIA_ROOT,
                                     forum_settings.ATTACHMENT_UPLOAD_TO)
        if not os.path.isdir(djangobb_path):
            msg = (
                "DjangoBB attachment path '%s' doesn't exists!"
                " Please check MEDIA_ROOT + ATTACHMENT_UPLOAD_TO in your local_settings.py!"
            ) % djangobb_path
            raise CommandError(msg)

    def check_models(self):
        has_entries = False
        for ModelClass in (Category, Forum, Profile, Post, Topic, Attachment):
            count = ModelClass.objects.all().count()
            if count != 0:
                has_entries = True
                self.err("ERROR: '%s' model has %i entries!\n" %
                         (ModelClass.__name__, count))
        if has_entries:
            self.warn("Maybe you have missed to use '--flush_djangobb' ?\n")
            if self.interactive:
                confirm = raw_input("\nContinue? (yes/no): ")
                while 1:
                    if confirm.lower().startswith("n"):
                        sys.exit("-1")
                    if confirm.lower() != "yes":
                        confirm = raw_input(
                            'Please enter either "yes" or "no": ')
                        continue
                    break

    def migrate_users(self, cleanup_users, moderator_groups):
        self.out(u"\n *** Migrate phpbb_forum users...\n")

        lang_codes = [i[0] for i in settings.LANGUAGES]
        default_lang = settings.LANGUAGE_CODE.split("-")[0]

        moderators = []
        user_dict = {}
        phpbb_users = phpbb_User.objects.all()

        total = phpbb_users.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        skip_count = 0
        start_time = time.time()
        next_status = start_time + 0.25
        for count, phpbb_user in enumerate(phpbb_users, 1):
            if self.max_entries and count >= self.max_entries:
                self.warn("Skip rest of users because max_entries used!")
                break

            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i users migrated %i skiped... rest: %i - eta: %s (rate: %.1f/sec)"
                    % (count, total, skip_count, rest, eta, rate))

            if not phpbb_user.posts:
                #assert phpbb_user.has_content() == False

                # Only users with has no posts can be skip.
                if cleanup_users >= 1:
                    if not phpbb_user.email:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no email)" %
                                smart_unicode(phpbb_user.username))
                        continue
                if cleanup_users >= 2:
                    if not phpbb_user.lastvisit:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no lastvisit)" %
                                smart_unicode(phpbb_user.username))
                        continue
                if cleanup_users >= 3:
                    if not phpbb_user.posts:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no posts)" %
                                smart_unicode(phpbb_user.username))
                        continue

            last_login = phpbb_user.lastvisit_datetime()
            if not last_login:
                # can't be None in User model:
                last_login = datetime.datetime(year=datetime.MINYEAR,
                                               month=1,
                                               day=1)

            is_moderator = phpbb_user.group in moderator_groups
            is_active = phpbb_user.inactive_time == 0

            # Prefix the password, e.g.: "$H$9dwN..." -> "phpBB3_md5$H$9dwN..."
            phpbb_password = PHPBB_PASSWORD % phpbb_user.password

            django_user, created = User.objects.get_or_create(
                username=phpbb_user.username,
                defaults={
                    "email": phpbb_user.email,
                    "password": phpbb_password,
                    "is_staff": is_moderator,
                    "is_active": is_active,
                    "is_superuser": False,
                    "last_login": last_login,
                    "date_joined": phpbb_user.registration_datetime(),
                })
            if created:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\tUser '%s' created." %
                                       smart_unicode(django_user.username))
            else:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\tUser '%s' exists." %
                                       smart_unicode(django_user.username))
                if django_user.password in (None, "", "!"):
                    # Update old migration
                    django_user.password = phpbb_password
                django_user.is_staff = is_moderator
                django_user.is_active = is_active
                django_user.save()

            if is_moderator:
                if self.verbosity >= 1:
                    self.out_overwrite(
                        u"\t *** Mark user '%s' as global forum moderator" %
                        phpbb_user)
                moderators.append(django_user)

            user_dict[phpbb_user.id] = django_user

            tz = TZ_CHOICES[int(phpbb_user.timezone)][0]
            #print tz

            # TODO: migrate avatar, too!
            # see:
            # https://github.com/jedie/django-phpBB3/issues/6

            if phpbb_user.lang in lang_codes:
                language = phpbb_user.lang
            else:
                language = default_lang

            cleaned_signature = phpbb_user.get_cleaned_signature()

            user_profile, created = Profile.objects.get_or_create(
                user=django_user,
                defaults={
                    "site": phpbb_user.website,
                    "signature": cleaned_signature,
                    #"signature_html": phpbb_user.sig,
                    "post_count": phpbb_user.posts,
                    "yahoo": phpbb_user.yim,
                    "time_zone": tz,
                    "language": language,
                    "jabber": phpbb_user.jabber,
                    "icq": phpbb_user.icq,
                    "msn": phpbb_user.msnm,
                    "aim": phpbb_user.aim,
                })
            if created:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\t - User profile for '%s' created." %
                                       smart_unicode(django_user.username))
            else:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\t - User profile for '%s' exists." %
                                       smart_unicode(django_user.username))

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(
            u" *** %i users migrated %i skiped in %s (rate: %.1f/sec)" %
            (count, skip_count, human_duration(duration), rate))

        return user_dict, moderators

    def get_or_create_category(self, phpbb_forum):
        obj, created = Category.objects.get_or_create(
            name=smart_unicode(phpbb_forum.forum_name))
        if created:
            self.out(u"\tCategory '%s' created.\n" % obj.name)
        else:
            self.out(u"\tCategory '%s' exists.\n" % obj.name)
        return obj

    def migrate_forums(self, moderators):
        self.out(u"\n *** Migrate phpbb_forum entries...\n")

        phpbb_forums = phpbb_Forum.objects.all()

        # Create categories
        category_dict = {}
        forum_dict = {}
        for phpbb_forum in phpbb_forums:
            #print phpbb_forum
            try:
                phpbb_forum.parent
            except phpbb_Forum.DoesNotExist:
                # has no parent -> is a Category
                # phpbb_forum.parent == 0 and a db item with ID 0 doesn't exist
                category = self.get_or_create_category(phpbb_forum)
                category_dict[phpbb_forum.id] = category
            else:
                # Has parent -> no Category
                continue

        # Create forums and categories for a sub forum
        for phpbb_forum in phpbb_forums:
            #print phpbb_forum
            try:
                # XXX: We can also use "forum_type"
                parent = phpbb_forum.parent
            except phpbb_Forum.DoesNotExist:
                # has no parent -> is a Category
                # skip, was created above
                continue

            try:
                category = category_dict[parent.id]
            except KeyError:
                # Create a new category for sub-forum
                parent = phpbb_Forum.objects.get(pk=parent.id)
                category = self.get_or_create_category(parent)
                category_dict[parent.id] = category

            obj, created = Forum.objects.get_or_create(
                name=smart_unicode(phpbb_forum.forum_name),
                defaults={
                    "category": category,
                    "description": phpbb_forum.forum_desc,

                    # TODO: set 'position' by flattern the btree

                    # These attributes would be set later in update_forum_stats():
                    # post_count, last_post, topic_count
                })
            if created:
                self.out(u"\tForum '%s' created.\n" % smart_unicode(obj.name))
            else:
                self.out(u"\tForum '%s' exists.\n" % smart_unicode(obj.name))

            forum_dict[phpbb_forum.id] = obj

            for moderator in moderators:
                obj.moderators.add(moderator)
            obj.save()
            self.out(u"\t - moderators: %s\n" % obj.moderators.all())

        return forum_dict

    _anonymous_user = None

    def _get_anonymous_user(self):
        """
        Pseudo account from phpBB
        Used to assign user to topics/posts from deleted users
        """
        if self._anonymous_user is None:
            self._anonymous_user, created = User.objects.get_or_create(# Pseudo account from phpBB
                username="******",
                defaults={
                    "password": UNUSABLE_PASSWORD,
                    "is_superuser": False,
                    "is_staff": False,
                    "is_active": False,
                }
            )
            if created:
                self.out(u"Anonymous User created")
        return self._anonymous_user

    def migrate_topic(self, user_dict, forum_dict):
        self.out(u"\n *** Migrate phpBB topic entries...\n")

        self.out(u"\tget topic watch information...")
        self.stdout.flush()
        topic_watch = get_topic_watch()
        self.out(u"OK\n")
        self.stdout.flush()

        anonymous_user = self._get_anonymous_user(
        )  # Pseudo account from phpBB

        topics = phpbb_Topic.objects.all().order_by("time")
        total = topics.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        start_time = time.time()
        next_status = start_time + 0.25
        for count, topic in enumerate(topics, 1):
            if self.max_entries and count >= self.max_entries:
                self.warn("Skip rest of topics because max_entries used!")
                break

            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i topics migrated... rest: %i - eta: %s (rate: %.1f/sec)"
                    % (count, total, rest, eta, rate))

            if topic.moved():
                # skip moved topics -> DjangoBB doesn't support them
                continue

            phpbb_user_id = topic.poster_id  #topic.poster.id
            try:
                user = user_dict[phpbb_user_id]
            except KeyError:
                self.out_overwrite(
                    self.style.NOTICE(
                        "topic %i poster: phpBB User with ID %i doesn't exist. Use Anonymous."
                        % (topic.id, phpbb_user_id)))
                user = anonymous_user

            forum = forum_dict[topic.forum_id]

            if topic.type in (1, 2, 3):
                # convert sticky, announce and global post to sticky
                # 0 == NORMAL, 1 == STICKY, 2 == ANNOUNCE, 3 == GLOBAL
                sticky = True
            else:
                sticky = False

            obj = Topic.objects.create(
                id=topic.id,
                forum=forum,
                user=user,
                name=topic.clean_title(),
                created=topic.create_datetime(),
                views=topic.views,
                sticky=sticky,
                closed=topic.locked(),

                # These attributes would be set later in update_topic_stats():
                # updated, post_count, last_post
            )
            if topic.id in topic_watch:
                subscribers = []
                phpbb_user_ids = topic_watch[topic.id]
                for phpbb_user_id in phpbb_user_ids:
                    try:
                        user = user_dict[phpbb_user_id]
                    except KeyError:
                        continue  # Skip not existing users.
                    subscribers.append(user)

                obj.subscribers = subscribers
                obj.save()

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(" *** %i topics migrated in %s (rate: %.1f/sec)" %
                           (count, human_duration(duration), rate))

    def migrate_posts(self, user_dict):
        self.out(u"\n *** Migrate phpBB posts entries...\n")

        anonymous_user = self._get_anonymous_user(
        )  # Pseudo account from phpBB

        posts = phpbb_Post.objects.all().order_by("time")
        total = posts.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        start_time = time.time()
        next_status = start_time + 0.25
        for count, phpbb_post in enumerate(posts, 1):
            if self.max_entries and count >= self.max_entries:
                self.warn("Skip rest of posts because max_entries used!")
                break

            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i posts migrated... rest: %i - eta: %s (rate: %.1f/sec)"
                    % (count, total, rest, eta, rate))

            topic_id = phpbb_post.topic_id
            try:
                topic = Topic.objects.get(id=topic_id)
            except Topic.DoesNotExist:
                self.out_overwrite(
                    self.style.NOTICE(
                        "topic for post %i doesn't exist! Skip post." %
                        phpbb_post.id))
                continue

            phpbb_user_id = phpbb_post.poster_id
            try:
                user = user_dict[phpbb_user_id]
            except KeyError:
                self.out_overwrite(
                    self.style.NOTICE(
                        "phpBB User with ID %i doesn't exist for post %i. Use Anonymous."
                        % (phpbb_user_id, phpbb_post.id)))
                user = anonymous_user

            if phpbb_post.edit_user > 0 and phpbb_post.edit_time > 0:
                updated = phpbb_post.update_datetime()
                try:
                    updated_by = user_dict[phpbb_post.edit_user]
                except KeyError:
                    updated_by = anonymous_user
            else:
                updated = None
                updated_by = None

            try:
                post = Post.objects.create(
                    id=phpbb_post.id,
                    topic=topic,
                    user=user,
                    created=phpbb_post.create_datetime(),
                    updated=updated,
                    updated_by=updated_by,
                    markup="bbcode",
                    body=phpbb_post.get_cleaned_bbcode(),
                    #body_html=html, # would be generated in save()
                    user_ip=phpbb_post.poster_ip,
                )
            except Exception, err:
                raise
                msg = (
                    "\n +++ ERROR: creating Post entry for phpBB3 post (ID: %s):\n"
                    "%s\n") % (phpbb_post.id, err)
                self.out_overwrite(self.style.NOTICE(msg))
                continue

            if phpbb_post.has_attachment():
                # copy attachment files
                phpbb_attachments = phpbb_Attachment.objects.filter(
                    post_msg=phpbb_post)
                for phpbb_attachment in phpbb_attachments:
                    src_path = os.path.join(settings.PHPBB_ATTACHMENT_PATH,
                                            phpbb_attachment.physical_filename)
                    if not os.path.isfile(src_path):
                        self.warn(
                            "\r\n +++ ERROR: Attachment not found: '%s'\n" %
                            src_path)
                    else:
                        attachment = Attachment(
                            size=phpbb_attachment.filesize,
                            content_type=phpbb_attachment.mimetype,
                            name=phpbb_attachment.real_filename,
                            post=post)
                        filename = "%d.0" % post.id
                        dst_path = os.path.join(
                            settings.MEDIA_ROOT,
                            forum_settings.ATTACHMENT_UPLOAD_TO, filename)
                        shutil.copy(src_path, dst_path)
                        attachment.path = filename
                        attachment.save()
                        self.out_overwrite(
                            "\t *** Attachment %s copied in: %s" %
                            (attachment.name, dst_path))

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(" *** %i posts migrated in %s (rate: %.1f/sec)" %
                           (count, human_duration(duration), rate))
コード例 #4
0
    def migrate_topic(self, user_dict, forum_dict):
        self.out(u"\n *** Migrate phpBB topic entries...\n")

        self.out(u"\tget topic watch information...")
        self.stdout.flush()
        topic_watch = get_topic_watch()
        self.out(u"OK\n")
        self.stdout.flush()

        anonymous_user = self._get_anonymous_user(
        )  # Pseudo account from phpBB

        topics = phpbb_Topic.objects.all().order_by("time")
        total = topics.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        start_time = time.time()
        next_status = start_time + 0.25
        for count, topic in enumerate(topics, 1):
            if self.max_entries and count >= self.max_entries:
                self.warn("Skip rest of topics because max_entries used!")
                break

            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i topics migrated... rest: %i - eta: %s (rate: %.1f/sec)"
                    % (count, total, rest, eta, rate))

            if topic.moved():
                # skip moved topics -> DjangoBB doesn't support them
                continue

            phpbb_user_id = topic.poster_id  #topic.poster.id
            try:
                user = user_dict[phpbb_user_id]
            except KeyError:
                self.out_overwrite(
                    self.style.NOTICE(
                        "topic %i poster: phpBB User with ID %i doesn't exist. Use Anonymous."
                        % (topic.id, phpbb_user_id)))
                user = anonymous_user

            forum = forum_dict[topic.forum_id]

            if topic.type in (1, 2, 3):
                # convert sticky, announce and global post to sticky
                # 0 == NORMAL, 1 == STICKY, 2 == ANNOUNCE, 3 == GLOBAL
                sticky = True
            else:
                sticky = False

            obj = Topic.objects.create(
                id=topic.id,
                forum=forum,
                user=user,
                name=topic.clean_title(),
                created=topic.create_datetime(),
                views=topic.views,
                sticky=sticky,
                closed=topic.locked(),

                # These attributes would be set later in update_topic_stats():
                # updated, post_count, last_post
            )
            if topic.id in topic_watch:
                subscribers = []
                phpbb_user_ids = topic_watch[topic.id]
                for phpbb_user_id in phpbb_user_ids:
                    try:
                        user = user_dict[phpbb_user_id]
                    except KeyError:
                        continue  # Skip not existing users.
                    subscribers.append(user)

                obj.subscribers = subscribers
                obj.save()

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(" *** %i topics migrated in %s (rate: %.1f/sec)" %
                           (count, human_duration(duration), rate))
コード例 #5
0
    def migrate_users(self, cleanup_users, moderator_groups):
        self.out(u"\n *** Migrate phpbb_forum users...\n")

        lang_codes = [i[0] for i in settings.LANGUAGES]
        default_lang = settings.LANGUAGE_CODE.split("-")[0]

        moderators = []
        user_dict = {}
        phpbb_users = phpbb_User.objects.all()

        total = phpbb_users.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        skip_count = 0
        start_time = time.time()
        next_status = start_time + 0.25
        for count, phpbb_user in enumerate(phpbb_users, 1):
            if self.max_entries and count >= self.max_entries:
                self.warn("Skip rest of users because max_entries used!")
                break

            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i users migrated %i skiped... rest: %i - eta: %s (rate: %.1f/sec)"
                    % (count, total, skip_count, rest, eta, rate))

            if not phpbb_user.posts:
                #assert phpbb_user.has_content() == False

                # Only users with has no posts can be skip.
                if cleanup_users >= 1:
                    if not phpbb_user.email:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no email)" %
                                smart_unicode(phpbb_user.username))
                        continue
                if cleanup_users >= 2:
                    if not phpbb_user.lastvisit:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no lastvisit)" %
                                smart_unicode(phpbb_user.username))
                        continue
                if cleanup_users >= 3:
                    if not phpbb_user.posts:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no posts)" %
                                smart_unicode(phpbb_user.username))
                        continue

            last_login = phpbb_user.lastvisit_datetime()
            if not last_login:
                # can't be None in User model:
                last_login = datetime.datetime(year=datetime.MINYEAR,
                                               month=1,
                                               day=1)

            is_moderator = phpbb_user.group in moderator_groups
            is_active = phpbb_user.inactive_time == 0

            # Prefix the password, e.g.: "$H$9dwN..." -> "phpBB3_md5$H$9dwN..."
            phpbb_password = PHPBB_PASSWORD % phpbb_user.password

            django_user, created = User.objects.get_or_create(
                username=phpbb_user.username,
                defaults={
                    "email": phpbb_user.email,
                    "password": phpbb_password,
                    "is_staff": is_moderator,
                    "is_active": is_active,
                    "is_superuser": False,
                    "last_login": last_login,
                    "date_joined": phpbb_user.registration_datetime(),
                })
            if created:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\tUser '%s' created." %
                                       smart_unicode(django_user.username))
            else:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\tUser '%s' exists." %
                                       smart_unicode(django_user.username))
                if django_user.password in (None, "", "!"):
                    # Update old migration
                    django_user.password = phpbb_password
                django_user.is_staff = is_moderator
                django_user.is_active = is_active
                django_user.save()

            if is_moderator:
                if self.verbosity >= 1:
                    self.out_overwrite(
                        u"\t *** Mark user '%s' as global forum moderator" %
                        phpbb_user)
                moderators.append(django_user)

            user_dict[phpbb_user.id] = django_user

            tz = TZ_CHOICES[int(phpbb_user.timezone)][0]
            #print tz

            # TODO: migrate avatar, too!
            # see:
            # https://github.com/jedie/django-phpBB3/issues/6

            if phpbb_user.lang in lang_codes:
                language = phpbb_user.lang
            else:
                language = default_lang

            cleaned_signature = phpbb_user.get_cleaned_signature()

            user_profile, created = Profile.objects.get_or_create(
                user=django_user,
                defaults={
                    "site": phpbb_user.website,
                    "signature": cleaned_signature,
                    #"signature_html": phpbb_user.sig,
                    "post_count": phpbb_user.posts,
                    "yahoo": phpbb_user.yim,
                    "time_zone": tz,
                    "language": language,
                    "jabber": phpbb_user.jabber,
                    "icq": phpbb_user.icq,
                    "msn": phpbb_user.msnm,
                    "aim": phpbb_user.aim,
                })
            if created:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\t - User profile for '%s' created." %
                                       smart_unicode(django_user.username))
            else:
                if self.verbosity >= 2:
                    self.out_overwrite(u"\t - User profile for '%s' exists." %
                                       smart_unicode(django_user.username))

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(
            u" *** %i users migrated %i skiped in %s (rate: %.1f/sec)" %
            (count, skip_count, human_duration(duration), rate))

        return user_dict, moderators
コード例 #6
0
    def migrate_topic(self, user_dict, forum_dict):
        self.stdout.write("\n *** Migrate phpBB topic entries...\n")

        self.stdout.write("\tget topic watch information...")
        self.stdout.flush()
        topic_watch = get_topic_watch()
        self.stdout.write("OK\n")
        self.stdout.flush()

        topic_dict = {}
        topics = phpbb_Topic.objects.all().order_by("time")
        total = topics.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        count = 0
        start_time = time.time()
        next_status = start_time + 0.25
        for topic in topics:
            count += 1
            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                msg = (
                    "\r\t%i/%i topics migrated... rest: %i - eta: %s (rate: %.1f/sec)         "
                ) % (count, total, rest, eta, rate)
                self.stdout.write(msg)
                self.stdout.flush()

            if topic.moved():
                # skip moved topics -> DjangoBB doesn't support them
                continue

            user = user_dict[topic.poster.id]
            forum = forum_dict[topic.forum.id]

            if topic.type in (1, 2, 3):
                # convert sticky, announce and global post to sticky
                # 0 == NORMAL, 1 == STICKY, 2 == ANNOUNCE, 3 == GLOBAL
                sticky = True
            else:
                sticky = False

            obj = Topic.objects.create(
                id=topic.id,
                forum=forum,
                user=user,
                name=topic.clean_title(),
                created=topic.create_datetime(),
                views=topic.views,
                sticky=sticky,
                closed=topic.locked(),

                # These attributes would be set later in update_topic_stats():
                # updated, post_count, last_post
            )
            if topic.id in topic_watch:
                subscribers = [user_dict[user_id] for user_id in topic_watch[topic.id]]
                obj.subscribers = subscribers
                obj.save()

            topic_dict[topic.id] = obj

        duration = time.time() - start_time
        rate = float(count) / duration
        self.stdout.write(
            "\r *** %i topics migrated in %s (rate: %.1f/sec)\n" % (
                count, human_duration(duration), rate
            )
        )
        return topic_dict
コード例 #7
0
                        )
                        shutil.copy(src_path, dst_path)
                        attachment.path = filename
                        attachment.save()
                        self.stdout.write(
                            "\n\t *** Attachment %s copied in: %s\n" % (
                                attachment.name, dst_path
                            )
                        )
                        self.stdout.flush()

        duration = time.time() - start_time
        rate = float(count) / duration
        self.stdout.write(
            "\r *** %i posts migrated in %s (rate: %.1f/sec)\n" % (
                count, human_duration(duration), rate
            )
        )

    def update_topic_stats(self):
        self.stdout.write(u"\n *** set topic stats...\n")

        topics = Topic.objects.all()
        total = topics.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        start_time = time.time()
        next_status = time.time() + 0.25
        for count, topic in enumerate(topics):
            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
コード例 #8
0
    def migrate_topic(self, user_dict, forum_dict):
        self.out(u"\n *** Migrate phpBB topic entries...\n")

        self.out(u"\tget topic watch information...")
        self.stdout.flush()
        topic_watch = get_topic_watch()
        self.out(u"OK\n")
        self.stdout.flush()

        anonymous_user = self._get_anonymous_user() # Pseudo account from phpBB

        topics = phpbb_Topic.objects.all().order_by("time")
        total = topics.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        start_time = time.time()
        next_status = start_time + 0.25
        for count, topic in enumerate(topics, 1):
            if self.max_entries and count >= self.max_entries:
                self.warn("Skip rest of topics because max_entries used!")
                break

            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i topics migrated... rest: %i - eta: %s (rate: %.1f/sec)" % (
                        count, total, rest, eta, rate
                    )
                )

            if topic.moved():
                # skip moved topics -> DjangoBB doesn't support them
                continue

            phpbb_user_id = topic.poster_id #topic.poster.id
            try:
                user = user_dict[phpbb_user_id]
            except KeyError:
                self.out_overwrite(self.style.NOTICE(
                    "topic %i poster: phpBB User with ID %i doesn't exist. Use Anonymous." % (
                        topic.id, phpbb_user_id
                    )
                ))
                user = anonymous_user

            forum = forum_dict[topic.forum_id]

            if topic.type in (1, 2, 3):
                # convert sticky, announce and global post to sticky
                # 0 == NORMAL, 1 == STICKY, 2 == ANNOUNCE, 3 == GLOBAL
                sticky = True
            else:
                sticky = False

            obj = Topic.objects.create(
                id=topic.id,
                forum=forum,
                user=user,
                name=topic.clean_title(),
                created=topic.create_datetime(),
                views=topic.views,
                sticky=sticky,
                closed=topic.locked(),

                # These attributes would be set later in update_topic_stats():
                # updated, post_count, last_post
            )
            if topic.id in topic_watch:
                subscribers = []
                phpbb_user_ids = topic_watch[topic.id]
                for phpbb_user_id in phpbb_user_ids:
                    try:
                        user = user_dict[phpbb_user_id]
                    except KeyError:
                        continue # Skip not existing users.
                    subscribers.append(user)

                obj.subscribers = subscribers
                obj.save()

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(
            " *** %i topics migrated in %s (rate: %.1f/sec)" % (
                count, human_duration(duration), rate
            )
        )
コード例 #9
0
    def migrate_users(self, cleanup_users, moderator_groups):
        self.out(u"\n *** Migrate phpbb_forum users...\n")

        lang_codes = [i[0] for i in settings.LANGUAGES]
        default_lang = settings.LANGUAGE_CODE.split("-")[0]

        moderators = []
        user_dict = {}
        phpbb_users = phpbb_User.objects.all()

        total = phpbb_users.count()
        process_info = ProcessInfo(total, use_last_rates=4)
        skip_count = 0
        start_time = time.time()
        next_status = start_time + 0.25
        for count, phpbb_user in enumerate(phpbb_users, 1):
            if self.max_entries and count >= self.max_entries:
                self.warn("Skip rest of users because max_entries used!")
                break

            if time.time() > next_status:
                next_status = time.time() + 1
                rest, eta, rate = process_info.update(count)
                self.out_update(
                    "\t%i/%i users migrated %i skiped... rest: %i - eta: %s (rate: %.1f/sec)" % (
                        count, total, skip_count, rest, eta, rate
                    )
                )

            if not phpbb_user.posts:
                #assert phpbb_user.has_content() == False

                # Only users with has no posts can be skip.
                if cleanup_users >= 1:
                    if not phpbb_user.email:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no email)" % smart_unicode(phpbb_user.username)
                            )
                        continue
                if cleanup_users >= 2:
                    if not phpbb_user.lastvisit:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no lastvisit)" % smart_unicode(phpbb_user.username)
                            )
                        continue
                if cleanup_users >= 3:
                    if not phpbb_user.posts:
                        skip_count += 1
                        if self.verbosity >= 2:
                            self.out_overwrite(
                                u"\t * Skip '%s' (no posts)" % smart_unicode(phpbb_user.username)
                            )
                        continue

            last_login = phpbb_user.lastvisit_datetime()
            if not last_login:
                # can't be None in User model:
                last_login = datetime.datetime(year=datetime.MINYEAR, month=1, day=1)

            is_moderator = phpbb_user.group in moderator_groups
            is_active = phpbb_user.inactive_time == 0


            # Prefix the password, e.g.: "$H$9dwN..." -> "phpBB3_md5$H$9dwN..."
            phpbb_password = PHPBB_PASSWORD % phpbb_user.password

            django_user, created = User.objects.get_or_create(
                username=phpbb_user.username,
                defaults={
                    "email":phpbb_user.email,
                    "password": phpbb_password,
                    "is_staff": is_moderator,
                    "is_active": is_active,
                    "is_superuser": False,
                    "last_login": last_login,
                    "date_joined": phpbb_user.registration_datetime(),
                }
            )
            if created:
                if self.verbosity >= 2:
                    self.out_overwrite(
                        u"\tUser '%s' created." % smart_unicode(django_user.username)
                    )
            else:
                if self.verbosity >= 2:
                    self.out_overwrite(
                        u"\tUser '%s' exists." % smart_unicode(django_user.username)
                    )
                if django_user.password in (None, "", "!"):
                    # Update old migration
                    django_user.password = phpbb_password
                django_user.is_staff = is_moderator
                django_user.is_active = is_active
                django_user.save()

            if is_moderator:
                if self.verbosity >= 1:
                    self.out_overwrite(
                        u"\t *** Mark user '%s' as global forum moderator" % phpbb_user
                    )
                moderators.append(django_user)

            user_dict[phpbb_user.id] = django_user

            tz = TZ_CHOICES[int(phpbb_user.timezone)][0]
            #print tz

            # TODO: migrate avatar, too!
            # see:
            # https://github.com/jedie/django-phpBB3/issues/6

            if phpbb_user.lang in lang_codes:
                language = phpbb_user.lang
            else:
                language = default_lang

            cleaned_signature = phpbb_user.get_cleaned_signature()

            user_profile, created = Profile.objects.get_or_create(
                user=django_user,
                defaults={
                    "site": phpbb_user.website,
                    "signature": cleaned_signature,
                    #"signature_html": phpbb_user.sig,
                    "post_count": phpbb_user.posts,
                    "yahoo": phpbb_user.yim,

                    "time_zone": tz,
                    "language":language,

                    "jabber":phpbb_user.jabber,
                    "icq":phpbb_user.icq,
                    "msn":phpbb_user.msnm,
                    "aim":phpbb_user.aim,
                }
            )
            if created:
                if self.verbosity >= 2:
                    self.out_overwrite(
                        u"\t - User profile for '%s' created." % smart_unicode(django_user.username)
                    )
            else:
                if self.verbosity >= 2:
                    self.out_overwrite(
                        u"\t - User profile for '%s' exists." % smart_unicode(django_user.username)
                    )

        duration = time.time() - start_time
        rate = float(count) / duration
        self.out_overwrite(
            u" *** %i users migrated %i skiped in %s (rate: %.1f/sec)" % (
                count, skip_count, human_duration(duration), rate
            )
        )

        return user_dict, moderators