def transfer_question_and_answer_activity(self): """transfers all question and answer edits and related status changes """ #assuming that there are only two post types se_revs = se.PostHistory.objects.all() #assuming that chronologial order is correct and there #will be no problems of data integrity upon insertion of records se_revs = se_revs.order_by('creation_date', 'revision_guid') #todo: ignored fringe case - no revisions c_guid = se_revs[0].revision_guid c_group = [] #this loop groups revisions by revision id, then calls process function #for the revision grup (elementary revisions posted at once) for se_rev in se_revs.iterator(): if se_rev.revision_guid == c_guid: c_group.append(se_rev) else: self._process_post_revision_group(c_group) c_group = [] c_group.append(se_rev) c_guid = se_rev.revision_guid transaction.commit() if len(c_group) != 0: self._process_post_revision_group(c_group)
def transfer_QA_votes(self): for v in se.Post2Vote.objects.all().iterator(): vote_type = v.vote_type.name if not vote_type in X.vote_actions: continue if v.user is None: continue u = X.get_user(v.user) p = X.get_post(v.post) if p is None: continue m = X.vote_actions[vote_type] vote_method = getattr(askbot.User, m) vote_method(u, p, timestamp=v.creation_date, force=True) if v.deletion_date: vote_method( u, p, timestamp=v.deletion_date, cancel=True, force=True #force to avoid permission errors ) transaction.commit()
def transfer_QA_votes(self): for v in se.Post2Vote.objects.all().iterator(): vote_type = v.vote_type.name if not vote_type in X.vote_actions: continue if v.user is None: continue u = X.get_user(v.user) p = X.get_post(v.post) if p is None: continue m = X.vote_actions[vote_type] vote_method = getattr(askbot.User, m) vote_method( u, p, timestamp = v.creation_date, force = True ) if v.deletion_date: vote_method( u, p, timestamp = v.deletion_date, cancel=True, force = True#force to avoid permission errors ) transaction.commit()
def transfer_exercise_and_problem_activity(self): """transfers all exercise and problem edits and related status changes """ #assuming that there are only two post types se_revs = se.PostHistory.objects.all() #assuming that chronologial order is correct and there #will be no problems of data integrity upon insertion of records se_revs = se_revs.order_by('creation_date','revision_guid') #todo: ignored fringe case - no revisions c_guid = se_revs[0].revision_guid c_group = [] #this loop groups revisions by revision id, then calls process function #for the revision grup (elementary revisions posted at once) for se_rev in se_revs.iterator(): if se_rev.revision_guid == c_guid: c_group.append(se_rev) else: self._process_post_revision_group(c_group) c_group = [] c_group.append(se_rev) c_guid = se_rev.revision_guid transaction.commit() if len(c_group) != 0: self._process_post_revision_group(c_group)
def transfer_comment_votes(self): for v in se.Comment2Vote.objects.all().iterator(): vote_type = v.vote_type.name if vote_type not in ('UpMod', 'Offensive'): continue if v.user is None: continue p = X.get_post(v.post_comment) #could also check deletion date on the Comment2Vote object #instead of making get_post return None on KeyError inside if p is None: #may be a deleted post continue u = X.get_user(v.user) m = X.vote_actions[vote_type] vote_method = getattr(askbot.User, m) vote_method(u, p, timestamp=v.creation_date, force=True) transaction.commit()
def transfer_comment_votes(self): for v in se.Comment2Vote.objects.all().iterator(): vote_type = v.vote_type.name if vote_type not in ('UpMod', 'Offensive'): continue if v.user is None: continue p = X.get_post(v.post_comment) #could also check deletion date on the Comment2Vote object #instead of making get_post return None on KeyError inside if p is None:#may be a deleted post continue u = X.get_user(v.user) m = X.vote_actions[vote_type] vote_method = getattr(askbot.User, m) vote_method( u, p, timestamp = v.creation_date, force = True ) transaction.commit()
def handle(self, *arg, **kwarg): if not importer_is_ready(): raise CommandError( "Looks like stackexchange tables are not yet initialized in the database.\n" "Please, run command: \npython manage.py syncdb\n" "then import the data.") award_badges_signal.disconnect(award_badges) if len(arg) < 1 or not os.path.isfile(arg[0]): raise CommandError( 'Error: first argument must be a zip file with the SE forum data' ) self.zipfile = self.open_dump(arg[0]) #read the data into SE tables for item in xml_read_order: time_before = datetime.now() self.load_xml_file(item) transaction.commit() time_after = datetime.now() if DEBUGME == True: print time_after - time_before print HEAP.heap() #this is important so that when we clean up messages #automatically generated by the procedures below #we do not delete old messages #todo: unfortunately this may need to be redone #when we upgrade to django 1.2 and definitely by 1.4 when #the current message system will be replaced with the #django messages framework self.save_askbot_message_id_list() #transfer data into ASKBOT tables print 'Transferring users...', self.transfer_users() transaction.commit() print 'done.' print 'Transferring content edits...', sys.stdout.flush() self.transfer_question_and_answer_activity() transaction.commit() print 'done.' print 'Transferring view counts...', sys.stdout.flush() self.transfer_question_view_counts() transaction.commit() print 'done.' print 'Transferring comments...', sys.stdout.flush() self.transfer_comments() transaction.commit() print 'done.' print 'Transferring badges and badge awards...', sys.stdout.flush() self.transfer_badges() transaction.commit() print 'done.' print 'Transferring Q&A votes...', sys.stdout.flush() self.transfer_QA_votes() #includes favorites, accepts and flags transaction.commit() print 'done.' print 'Transferring comment votes...', sys.stdout.flush() self.transfer_comment_votes() transaction.commit() self.cleanup_messages() #delete autogenerated messages transaction.commit() self.transfer_messages() transaction.commit() #todo: these are not clear how to go about self.transfer_update_subscriptions() transaction.commit() self.transfer_tag_preferences() transaction.commit() self.transfer_meta_pages() transaction.commit() print 'done.'
def handle(self, *arg, **kwarg): if django_settings.DEBUG: raise CommandError( 'Please set DEBUG to False in the settings.py to reduce ' 'RAM usage during the import process') #process the command line arguments, if given if kwarg['read_dump'] is False and kwarg['process_data'] is False: #make them both true as a hack to simulate a condition where #no flags selected means the same as both are indeed selected kwarg['read_dump'] = True kwarg['process_data'] = True askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', False) if not importer_is_ready(): raise CommandError( "Looks like stackexchange tables are not yet initialized in the database.\n" "Please, run command: \npython manage.py syncdb\n" "then import the data.") award_badges_signal.disconnect(award_badges) if len(arg) < 1 or not os.path.isfile(arg[0]): raise CommandError( 'Error: first argument must be a zip file with the SE forum data' ) if kwarg['read_dump']: self.zipfile = self.open_dump(arg[0]) #read the data into SE tables for item in xml_read_order: time_before = timezone.now() self.load_xml_file(item) transaction.commit() time_after = timezone.now() if DEBUGME == True: print time_after - time_before print HEAP.heap() if kwarg['process_data'] is False: #that means we just wanted to load the xml dump to #do the second step in another go in order to have #more ram for the transfer of data from SE to Askbot databases return #this is important so that when we clean up messages #automatically generated by the procedures below #we do not delete old messages #todo: unfortunately this may need to be redone #when we upgrade to django 1.2 and definitely by 1.4 when #the current message system will be replaced with the #django messages framework self.save_askbot_message_id_list() #transfer data into ASKBOT tables print 'Transferring users...' self.transfer_users() transaction.commit() print 'done.' print 'Transferring content edits...' sys.stdout.flush() self.transfer_question_and_answer_activity() transaction.commit() print 'done.' print 'Transferring view counts...' sys.stdout.flush() self.transfer_question_view_counts() transaction.commit() print 'done.' print 'Transferring comments...' sys.stdout.flush() self.transfer_comments() transaction.commit() print 'done.' print 'Transferring badges and badge awards...' sys.stdout.flush() self.transfer_badges() transaction.commit() print 'done.' print 'Transferring Q&A votes...' sys.stdout.flush() self.transfer_QA_votes() #includes favorites, accepts and flags transaction.commit() print 'done.' print 'Transferring comment votes...' sys.stdout.flush() self.transfer_comment_votes() transaction.commit() self.cleanup_messages() #delete autogenerated messages transaction.commit() self.transfer_messages() transaction.commit() #todo: these are not clear how to go about self.transfer_update_subscriptions() transaction.commit() self.transfer_tag_preferences() transaction.commit() self.transfer_meta_pages() transaction.commit() print 'done.'
def handle(self, *arg, **kwarg): if not importer_is_ready(): raise CommandError( "Looks like stackexchange tables are not yet initialized in the database.\n" "Please, run command: \npython manage.py syncdb\n" "then import the data." ) award_badges_signal.disconnect(award_badges) if len(arg) < 1 or not os.path.isfile(arg[0]): raise CommandError('Error: first argument must be a zip file with the SE forum data') self.zipfile = self.open_dump(arg[0]) #read the data into SE tables for item in xml_read_order: time_before = datetime.now() self.load_xml_file(item) transaction.commit() time_after = datetime.now() if DEBUGME == True: print time_after - time_before print HEAP.heap() #this is important so that when we clean up messages #automatically generated by the procedures below #we do not delete old messages #todo: unfortunately this may need to be redone #when we upgrade to django 1.2 and definitely by 1.4 when #the current message system will be replaced with the #django messages framework self.save_askbot_message_id_list() #transfer data into ASKBOT tables print 'Transferring users...', self.transfer_users() transaction.commit() print 'done.' print 'Transferring content edits...', sys.stdout.flush() self.transfer_exercise_and_problem_activity() transaction.commit() print 'done.' print 'Transferring view counts...', sys.stdout.flush() self.transfer_exercise_view_counts() transaction.commit() print 'done.' print 'Transferring comments...', sys.stdout.flush() self.transfer_comments() transaction.commit() print 'done.' print 'Transferring badges and badge awards...', sys.stdout.flush() self.transfer_badges() transaction.commit() print 'done.' print 'Transferring Q&A votes...', sys.stdout.flush() self.transfer_QA_votes()#includes favorites, accepts and flags transaction.commit() print 'done.' print 'Transferring comment votes...', sys.stdout.flush() self.transfer_comment_votes() transaction.commit() self.cleanup_messages()#delete autogenerated messages transaction.commit() self.transfer_messages() transaction.commit() #todo: these are not clear how to go about self.transfer_update_subscriptions() transaction.commit() self.transfer_tag_preferences() transaction.commit() self.transfer_meta_pages() transaction.commit() print 'done.'
def handle(self, *arg, **kwarg): if django_settings.DEBUG: raise CommandError( 'Please set DEBUG to False in the settings.py to reduce ' 'RAM usage during the import process' ) #process the command line arguments, if given if kwarg['read_dump'] is False and kwarg['process_data'] is False: #make them both true as a hack to simulate a condition where #no flags selected means the same as both are indeed selected kwarg['read_dump'] = True kwarg['process_data'] = True askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', False) if not importer_is_ready(): raise CommandError( "Looks like stackexchange tables are not yet initialized in the database.\n" "Please, run command: \npython manage.py syncdb\n" "then import the data." ) award_badges_signal.disconnect(award_badges) if len(arg) < 1 or not os.path.isfile(arg[0]): raise CommandError('Error: first argument must be a zip file with the SE forum data') if kwarg['read_dump']: self.zipfile = self.open_dump(arg[0]) #read the data into SE tables for item in xml_read_order: time_before = datetime.now() self.load_xml_file(item) transaction.commit() time_after = datetime.now() if DEBUGME == True: print time_after - time_before print HEAP.heap() if kwarg['process_data'] is False: #that means we just wanted to load the xml dump to #do the second step in another go in order to have #more ram for the transfer of data from SE to Askbot databases return #this is important so that when we clean up messages #automatically generated by the procedures below #we do not delete old messages #todo: unfortunately this may need to be redone #when we upgrade to django 1.2 and definitely by 1.4 when #the current message system will be replaced with the #django messages framework self.save_askbot_message_id_list() #transfer data into ASKBOT tables print 'Transferring users...' self.transfer_users() transaction.commit() print 'done.' print 'Transferring content edits...' sys.stdout.flush() self.transfer_question_and_answer_activity() transaction.commit() print 'done.' print 'Transferring view counts...' sys.stdout.flush() self.transfer_question_view_counts() transaction.commit() print 'done.' print 'Transferring comments...' sys.stdout.flush() self.transfer_comments() transaction.commit() print 'done.' print 'Transferring badges and badge awards...' sys.stdout.flush() self.transfer_badges() transaction.commit() print 'done.' print 'Transferring Q&A votes...' sys.stdout.flush() self.transfer_QA_votes()#includes favorites, accepts and flags transaction.commit() print 'done.' print 'Transferring comment votes...' sys.stdout.flush() self.transfer_comment_votes() transaction.commit() self.cleanup_messages()#delete autogenerated messages transaction.commit() self.transfer_messages() transaction.commit() #todo: these are not clear how to go about self.transfer_update_subscriptions() transaction.commit() self.transfer_tag_preferences() transaction.commit() self.transfer_meta_pages() transaction.commit() print 'done.'