示例#1
0
    def handle(self, *arg, **kwarg):

        if not importer_is_ready():
            raise CommandError(
                "Looks like stackexchange tables are not yet initialized in the database.\n"
                "Please, run command: \npython manage.py syncdb\n"
                "then import the data.")

        award_badges_signal.disconnect(award_badges)

        if len(arg) < 1 or not os.path.isfile(arg[0]):
            raise CommandError(
                'Error: first argument must be a zip file with the SE forum data'
            )

        self.zipfile = self.open_dump(arg[0])
        #read the data into SE tables
        for item in xml_read_order:
            time_before = datetime.now()
            self.load_xml_file(item)
            transaction.commit()
            time_after = datetime.now()
            if DEBUGME == True:
                print time_after - time_before
                print HEAP.heap()

        #this is important so that when we clean up messages
        #automatically generated by the procedures below
        #we do not delete old messages
        #todo: unfortunately this may need to be redone
        #when we upgrade to django 1.2 and definitely by 1.4 when
        #the current message system will be replaced with the
        #django messages framework
        self.save_askbot_message_id_list()

        #transfer data into ASKBOT tables
        print 'Transferring users...',
        self.transfer_users()
        transaction.commit()
        print 'done.'
        print 'Transferring content edits...',
        sys.stdout.flush()
        self.transfer_question_and_answer_activity()
        transaction.commit()
        print 'done.'
        print 'Transferring view counts...',
        sys.stdout.flush()
        self.transfer_question_view_counts()
        transaction.commit()
        print 'done.'
        print 'Transferring comments...',
        sys.stdout.flush()
        self.transfer_comments()
        transaction.commit()
        print 'done.'
        print 'Transferring badges and badge awards...',
        sys.stdout.flush()
        self.transfer_badges()
        transaction.commit()
        print 'done.'
        print 'Transferring Q&A votes...',
        sys.stdout.flush()
        self.transfer_QA_votes()  #includes favorites, accepts and flags
        transaction.commit()
        print 'done.'
        print 'Transferring comment votes...',
        sys.stdout.flush()
        self.transfer_comment_votes()
        transaction.commit()

        self.cleanup_messages()  #delete autogenerated messages
        transaction.commit()
        self.transfer_messages()
        transaction.commit()

        #todo: these are not clear how to go about
        self.transfer_update_subscriptions()
        transaction.commit()
        self.transfer_tag_preferences()
        transaction.commit()
        self.transfer_meta_pages()
        transaction.commit()
        print 'done.'
示例#2
0
    def handle(self, *arg, **kwarg):

        if django_settings.DEBUG:
            raise CommandError(
                'Please set DEBUG to False in the settings.py to reduce '
                'RAM usage during the import process')

        #process the command line arguments, if given
        if kwarg['read_dump'] is False and kwarg['process_data'] is False:
            #make them both true as a hack to simulate a condition where
            #no flags selected means the same as both are indeed selected
            kwarg['read_dump'] = True
            kwarg['process_data'] = True

        askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', False)

        if not importer_is_ready():
            raise CommandError(
                "Looks like stackexchange tables are not yet initialized in the database.\n"
                "Please, run command: \npython manage.py syncdb\n"
                "then import the data.")

        award_badges_signal.disconnect(award_badges)

        if len(arg) < 1 or not os.path.isfile(arg[0]):
            raise CommandError(
                'Error: first argument must be a zip file with the SE forum data'
            )

        if kwarg['read_dump']:
            self.zipfile = self.open_dump(arg[0])
            #read the data into SE tables
            for item in xml_read_order:
                time_before = timezone.now()
                self.load_xml_file(item)
                transaction.commit()
                time_after = timezone.now()
                if DEBUGME == True:
                    print time_after - time_before
                    print HEAP.heap()

        if kwarg['process_data'] is False:
            #that means we just wanted to load the xml dump to
            #do the second step in another go in order to have
            #more ram for the transfer of data from SE to Askbot databases
            return

        #this is important so that when we clean up messages
        #automatically generated by the procedures below
        #we do not delete old messages
        #todo: unfortunately this may need to be redone
        #when we upgrade to django 1.2 and definitely by 1.4 when
        #the current message system will be replaced with the
        #django messages framework
        self.save_askbot_message_id_list()

        #transfer data into ASKBOT tables
        print 'Transferring users...'
        self.transfer_users()
        transaction.commit()
        print 'done.'
        print 'Transferring content edits...'
        sys.stdout.flush()
        self.transfer_question_and_answer_activity()
        transaction.commit()
        print 'done.'
        print 'Transferring view counts...'
        sys.stdout.flush()
        self.transfer_question_view_counts()
        transaction.commit()
        print 'done.'
        print 'Transferring comments...'
        sys.stdout.flush()
        self.transfer_comments()
        transaction.commit()
        print 'done.'
        print 'Transferring badges and badge awards...'
        sys.stdout.flush()
        self.transfer_badges()
        transaction.commit()
        print 'done.'
        print 'Transferring Q&A votes...'
        sys.stdout.flush()
        self.transfer_QA_votes()  #includes favorites, accepts and flags
        transaction.commit()
        print 'done.'
        print 'Transferring comment votes...'
        sys.stdout.flush()
        self.transfer_comment_votes()
        transaction.commit()

        self.cleanup_messages()  #delete autogenerated messages
        transaction.commit()
        self.transfer_messages()
        transaction.commit()

        #todo: these are not clear how to go about
        self.transfer_update_subscriptions()
        transaction.commit()
        self.transfer_tag_preferences()
        transaction.commit()
        self.transfer_meta_pages()
        transaction.commit()
        print 'done.'
    def handle(self, *arg, **kwarg):

        award_badges_signal.disconnect(award_badges)

        if len(arg) < 1 or not os.path.isdir(arg[0]):
            print 'Error: first argument must be a directory with all the SE *.xml files'
            sys.exit(1)

        self.dump_path = arg[0]
        #read the data into SE tables
        for xml in xml_read_order:
            xml_path = self.get_xml_path(xml)
            table_name = self.get_table_name(xml)
            self.load_xml_file(xml_path, table_name)
            transaction.commit()

        #this is important so that when we clean up messages
        #automatically generated by the procedures below
        #we do not delete old messages
        #todo: unfortunately this may need to be redone
        #when we upgrade to django 1.2 and definitely by 1.4 when
        #the current message system will be replaced with the
        #django messages framework
        self.save_askbot_message_id_list()

        #transfer data into ASKBOT tables
        print 'Transferring users...',
        sys.stdout.flush()
        self.transfer_users()
        transaction.commit()
        print 'done.'
        print 'Transferring content edits...',
        sys.stdout.flush()
        self.transfer_question_and_answer_activity()
        transaction.commit()
        print 'done.'
        print 'Transferring view counts...',
        sys.stdout.flush()
        self.transfer_question_view_counts()
        transaction.commit()
        print 'done.'
        print 'Transferring comments...',
        sys.stdout.flush()
        self.transfer_comments()
        transaction.commit()
        print 'done.'
        print 'Transferring badges and badge awards...',
        sys.stdout.flush()
        self.transfer_badges()
        transaction.commit()
        print 'done.'
        print 'Transferring votes...',
        sys.stdout.flush()
        self.transfer_votes()#includes favorites, accepts and flags
        transaction.commit()
        print 'done.'

        self.cleanup_messages()#delete autogenerated messages
        transaction.commit()
        self.transfer_messages()
        transaction.commit()

        #todo: these are not clear how to go about
        self.transfer_update_subscriptions()
        transaction.commit()
        self.transfer_tag_preferences()
        transaction.commit()
        self.transfer_meta_pages()
        transaction.commit()
示例#4
0
    def handle(self, *arg, **kwarg):

        if not importer_is_ready():
            raise CommandError(
                "Looks like stackexchange tables are not yet initialized in the database.\n"
                "Please, run command: \npython manage.py syncdb\n"
                "then import the data."
            )


        award_badges_signal.disconnect(award_badges)

        if len(arg) < 1 or not os.path.isfile(arg[0]):
            raise CommandError('Error: first argument must be a zip file with the SE forum data')

        self.zipfile = self.open_dump(arg[0]) 
        #read the data into SE tables
        for item in xml_read_order:
            time_before = datetime.now()
            self.load_xml_file(item)
            transaction.commit()
            time_after = datetime.now()
            if DEBUGME == True:
                print time_after - time_before
                print HEAP.heap()

        #this is important so that when we clean up messages
        #automatically generated by the procedures below
        #we do not delete old messages
        #todo: unfortunately this may need to be redone
        #when we upgrade to django 1.2 and definitely by 1.4 when
        #the current message system will be replaced with the
        #django messages framework
        self.save_askbot_message_id_list()

        #transfer data into ASKBOT tables
        print 'Transferring users...',
        self.transfer_users()
        transaction.commit()
        print 'done.'
        print 'Transferring content edits...',
        sys.stdout.flush()
        self.transfer_exercise_and_problem_activity()
        transaction.commit()
        print 'done.'
        print 'Transferring view counts...',
        sys.stdout.flush()
        self.transfer_exercise_view_counts()
        transaction.commit()
        print 'done.'
        print 'Transferring comments...',
        sys.stdout.flush()
        self.transfer_comments()
        transaction.commit()
        print 'done.'
        print 'Transferring badges and badge awards...',
        sys.stdout.flush()
        self.transfer_badges()
        transaction.commit()
        print 'done.'
        print 'Transferring Q&A votes...',
        sys.stdout.flush()
        self.transfer_QA_votes()#includes favorites, accepts and flags
        transaction.commit()
        print 'done.'
        print 'Transferring comment votes...',
        sys.stdout.flush()
        self.transfer_comment_votes()
        transaction.commit()

        self.cleanup_messages()#delete autogenerated messages
        transaction.commit()
        self.transfer_messages()
        transaction.commit()

        #todo: these are not clear how to go about
        self.transfer_update_subscriptions()
        transaction.commit()
        self.transfer_tag_preferences()
        transaction.commit()
        self.transfer_meta_pages()
        transaction.commit()
        print 'done.'
示例#5
0
    def handle(self, *arg, **kwarg):

        if django_settings.DEBUG:
            raise CommandError(
                'Please set DEBUG to False in the settings.py to reduce '
                'RAM usage during the import process'
            )

        #process the command line arguments, if given
        if kwarg['read_dump'] is False and kwarg['process_data'] is False:
            #make them both true as a hack to simulate a condition where
            #no flags selected means the same as both are indeed selected
            kwarg['read_dump'] = True
            kwarg['process_data'] = True

        askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', False)

        if not importer_is_ready():
            raise CommandError(
                "Looks like stackexchange tables are not yet initialized in the database.\n"
                "Please, run command: \npython manage.py syncdb\n"
                "then import the data."
            )


        award_badges_signal.disconnect(award_badges)

        if len(arg) < 1 or not os.path.isfile(arg[0]):
            raise CommandError('Error: first argument must be a zip file with the SE forum data')

        if kwarg['read_dump']:
            self.zipfile = self.open_dump(arg[0])
            #read the data into SE tables
            for item in xml_read_order:
                time_before = datetime.now()
                self.load_xml_file(item)
                transaction.commit()
                time_after = datetime.now()
                if DEBUGME == True:
                    print time_after - time_before
                    print HEAP.heap()

        if kwarg['process_data'] is False:
            #that means we just wanted to load the xml dump to
            #do the second step in another go in order to have
            #more ram for the transfer of data from SE to Askbot databases
            return

        #this is important so that when we clean up messages
        #automatically generated by the procedures below
        #we do not delete old messages
        #todo: unfortunately this may need to be redone
        #when we upgrade to django 1.2 and definitely by 1.4 when
        #the current message system will be replaced with the
        #django messages framework
        self.save_askbot_message_id_list()

        #transfer data into ASKBOT tables
        print 'Transferring users...'
        self.transfer_users()
        transaction.commit()
        print 'done.'
        print 'Transferring content edits...'
        sys.stdout.flush()
        self.transfer_question_and_answer_activity()
        transaction.commit()
        print 'done.'
        print 'Transferring view counts...'
        sys.stdout.flush()
        self.transfer_question_view_counts()
        transaction.commit()
        print 'done.'
        print 'Transferring comments...'
        sys.stdout.flush()
        self.transfer_comments()
        transaction.commit()
        print 'done.'
        print 'Transferring badges and badge awards...'
        sys.stdout.flush()
        self.transfer_badges()
        transaction.commit()
        print 'done.'
        print 'Transferring Q&A votes...'
        sys.stdout.flush()
        self.transfer_QA_votes()#includes favorites, accepts and flags
        transaction.commit()
        print 'done.'
        print 'Transferring comment votes...'
        sys.stdout.flush()
        self.transfer_comment_votes()
        transaction.commit()

        self.cleanup_messages()#delete autogenerated messages
        transaction.commit()
        self.transfer_messages()
        transaction.commit()

        #todo: these are not clear how to go about
        self.transfer_update_subscriptions()
        transaction.commit()
        self.transfer_tag_preferences()
        transaction.commit()
        self.transfer_meta_pages()
        transaction.commit()
        print 'done.'