def transfer_question_and_answer_activity(self):
"""transfers all question and answer
edits and related status changes
"""
#assuming that there are only two post types
se_revs = se.PostHistory.objects.all()
#assuming that chronologial order is correct and there
#will be no problems of data integrity upon insertion of records
se_revs = se_revs.order_by('creation_date', 'revision_guid')
#todo: ignored fringe case - no revisions
c_guid = se_revs[0].revision_guid
c_group = []
#this loop groups revisions by revision id, then calls process function
#for the revision grup (elementary revisions posted at once)
for se_rev in se_revs.iterator():
if se_rev.revision_guid == c_guid:
c_group.append(se_rev)
else:
self._process_post_revision_group(c_group)
c_group = []
c_group.append(se_rev)
c_guid = se_rev.revision_guid
transaction.commit()
if len(c_group) != 0:
self._process_post_revision_group(c_group)
def transfer_QA_votes(self):
for v in se.Post2Vote.objects.all().iterator():
vote_type = v.vote_type.name
if not vote_type in X.vote_actions:
continue
if v.user is None:
continue
u = X.get_user(v.user)
p = X.get_post(v.post)
if p is None:
continue
m = X.vote_actions[vote_type]
vote_method = getattr(askbot.User, m)
vote_method(u, p, timestamp=v.creation_date, force=True)
if v.deletion_date:
vote_method(
u,
p,
timestamp=v.deletion_date,
cancel=True,
force=True #force to avoid permission errors
)
transaction.commit()
def transfer_QA_votes(self):
for v in se.Post2Vote.objects.all().iterator():
vote_type = v.vote_type.name
if not vote_type in X.vote_actions:
continue
if v.user is None:
continue
u = X.get_user(v.user)
p = X.get_post(v.post)
if p is None:
continue
m = X.vote_actions[vote_type]
vote_method = getattr(askbot.User, m)
vote_method(
u, p, timestamp = v.creation_date,
force = True
)
if v.deletion_date:
vote_method(
u, p, timestamp = v.deletion_date,
cancel=True,
force = True#force to avoid permission errors
)
transaction.commit()
def transfer_exercise_and_problem_activity(self):
"""transfers all exercise and problem
edits and related status changes
"""
#assuming that there are only two post types
se_revs = se.PostHistory.objects.all()
#assuming that chronologial order is correct and there
#will be no problems of data integrity upon insertion of records
se_revs = se_revs.order_by('creation_date','revision_guid')
#todo: ignored fringe case - no revisions
c_guid = se_revs[0].revision_guid
c_group = []
#this loop groups revisions by revision id, then calls process function
#for the revision grup (elementary revisions posted at once)
for se_rev in se_revs.iterator():
if se_rev.revision_guid == c_guid:
c_group.append(se_rev)
else:
self._process_post_revision_group(c_group)
c_group = []
c_group.append(se_rev)
c_guid = se_rev.revision_guid
transaction.commit()
if len(c_group) != 0:
self._process_post_revision_group(c_group)
def transfer_comment_votes(self):
for v in se.Comment2Vote.objects.all().iterator():
vote_type = v.vote_type.name
if vote_type not in ('UpMod', 'Offensive'):
continue
if v.user is None:
continue
p = X.get_post(v.post_comment)
#could also check deletion date on the Comment2Vote object
#instead of making get_post return None on KeyError inside
if p is None: #may be a deleted post
continue
u = X.get_user(v.user)
m = X.vote_actions[vote_type]
vote_method = getattr(askbot.User, m)
vote_method(u, p, timestamp=v.creation_date, force=True)
transaction.commit()
def transfer_comment_votes(self):
for v in se.Comment2Vote.objects.all().iterator():
vote_type = v.vote_type.name
if vote_type not in ('UpMod', 'Offensive'):
continue
if v.user is None:
continue
p = X.get_post(v.post_comment)
#could also check deletion date on the Comment2Vote object
#instead of making get_post return None on KeyError inside
if p is None:#may be a deleted post
continue
u = X.get_user(v.user)
m = X.vote_actions[vote_type]
vote_method = getattr(askbot.User, m)
vote_method(
u, p, timestamp = v.creation_date,
force = True
)
transaction.commit()
def handle(self, *arg, **kwarg):
if not importer_is_ready():
raise CommandError(
"Looks like stackexchange tables are not yet initialized in the database.\n"
"Please, run command: \npython manage.py syncdb\n"
"then import the data.")
award_badges_signal.disconnect(award_badges)
if len(arg) < 1 or not os.path.isfile(arg[0]):
raise CommandError(
'Error: first argument must be a zip file with the SE forum data'
)
self.zipfile = self.open_dump(arg[0])
#read the data into SE tables
for item in xml_read_order:
time_before = datetime.now()
self.load_xml_file(item)
transaction.commit()
time_after = datetime.now()
if DEBUGME == True:
print time_after - time_before
print HEAP.heap()
#this is important so that when we clean up messages
#automatically generated by the procedures below
#we do not delete old messages
#todo: unfortunately this may need to be redone
#when we upgrade to django 1.2 and definitely by 1.4 when
#the current message system will be replaced with the
#django messages framework
self.save_askbot_message_id_list()
#transfer data into ASKBOT tables
print 'Transferring users...',
self.transfer_users()
transaction.commit()
print 'done.'
print 'Transferring content edits...',
sys.stdout.flush()
self.transfer_question_and_answer_activity()
transaction.commit()
print 'done.'
print 'Transferring view counts...',
sys.stdout.flush()
self.transfer_question_view_counts()
transaction.commit()
print 'done.'
print 'Transferring comments...',
sys.stdout.flush()
self.transfer_comments()
transaction.commit()
print 'done.'
print 'Transferring badges and badge awards...',
sys.stdout.flush()
self.transfer_badges()
transaction.commit()
print 'done.'
print 'Transferring Q&A votes...',
sys.stdout.flush()
self.transfer_QA_votes() #includes favorites, accepts and flags
transaction.commit()
print 'done.'
print 'Transferring comment votes...',
sys.stdout.flush()
self.transfer_comment_votes()
transaction.commit()
self.cleanup_messages() #delete autogenerated messages
transaction.commit()
self.transfer_messages()
transaction.commit()
#todo: these are not clear how to go about
self.transfer_update_subscriptions()
transaction.commit()
self.transfer_tag_preferences()
transaction.commit()
self.transfer_meta_pages()
transaction.commit()
print 'done.'
def handle(self, *arg, **kwarg):
if django_settings.DEBUG:
raise CommandError(
'Please set DEBUG to False in the settings.py to reduce '
'RAM usage during the import process')
#process the command line arguments, if given
if kwarg['read_dump'] is False and kwarg['process_data'] is False:
#make them both true as a hack to simulate a condition where
#no flags selected means the same as both are indeed selected
kwarg['read_dump'] = True
kwarg['process_data'] = True
askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', False)
if not importer_is_ready():
raise CommandError(
"Looks like stackexchange tables are not yet initialized in the database.\n"
"Please, run command: \npython manage.py syncdb\n"
"then import the data.")
award_badges_signal.disconnect(award_badges)
if len(arg) < 1 or not os.path.isfile(arg[0]):
raise CommandError(
'Error: first argument must be a zip file with the SE forum data'
)
if kwarg['read_dump']:
self.zipfile = self.open_dump(arg[0])
#read the data into SE tables
for item in xml_read_order:
time_before = timezone.now()
self.load_xml_file(item)
transaction.commit()
time_after = timezone.now()
if DEBUGME == True:
print time_after - time_before
print HEAP.heap()
if kwarg['process_data'] is False:
#that means we just wanted to load the xml dump to
#do the second step in another go in order to have
#more ram for the transfer of data from SE to Askbot databases
return
#this is important so that when we clean up messages
#automatically generated by the procedures below
#we do not delete old messages
#todo: unfortunately this may need to be redone
#when we upgrade to django 1.2 and definitely by 1.4 when
#the current message system will be replaced with the
#django messages framework
self.save_askbot_message_id_list()
#transfer data into ASKBOT tables
print 'Transferring users...'
self.transfer_users()
transaction.commit()
print 'done.'
print 'Transferring content edits...'
sys.stdout.flush()
self.transfer_question_and_answer_activity()
transaction.commit()
print 'done.'
print 'Transferring view counts...'
sys.stdout.flush()
self.transfer_question_view_counts()
transaction.commit()
print 'done.'
print 'Transferring comments...'
sys.stdout.flush()
self.transfer_comments()
transaction.commit()
print 'done.'
print 'Transferring badges and badge awards...'
sys.stdout.flush()
self.transfer_badges()
transaction.commit()
print 'done.'
print 'Transferring Q&A votes...'
sys.stdout.flush()
self.transfer_QA_votes() #includes favorites, accepts and flags
transaction.commit()
print 'done.'
print 'Transferring comment votes...'
sys.stdout.flush()
self.transfer_comment_votes()
transaction.commit()
self.cleanup_messages() #delete autogenerated messages
transaction.commit()
self.transfer_messages()
transaction.commit()
#todo: these are not clear how to go about
self.transfer_update_subscriptions()
transaction.commit()
self.transfer_tag_preferences()
transaction.commit()
self.transfer_meta_pages()
transaction.commit()
print 'done.'
def handle(self, *arg, **kwarg):
if not importer_is_ready():
raise CommandError(
"Looks like stackexchange tables are not yet initialized in the database.\n"
"Please, run command: \npython manage.py syncdb\n"
"then import the data."
)
award_badges_signal.disconnect(award_badges)
if len(arg) < 1 or not os.path.isfile(arg[0]):
raise CommandError('Error: first argument must be a zip file with the SE forum data')
self.zipfile = self.open_dump(arg[0])
#read the data into SE tables
for item in xml_read_order:
time_before = datetime.now()
self.load_xml_file(item)
transaction.commit()
time_after = datetime.now()
if DEBUGME == True:
print time_after - time_before
print HEAP.heap()
#this is important so that when we clean up messages
#automatically generated by the procedures below
#we do not delete old messages
#todo: unfortunately this may need to be redone
#when we upgrade to django 1.2 and definitely by 1.4 when
#the current message system will be replaced with the
#django messages framework
self.save_askbot_message_id_list()
#transfer data into ASKBOT tables
print 'Transferring users...',
self.transfer_users()
transaction.commit()
print 'done.'
print 'Transferring content edits...',
sys.stdout.flush()
self.transfer_exercise_and_problem_activity()
transaction.commit()
print 'done.'
print 'Transferring view counts...',
sys.stdout.flush()
self.transfer_exercise_view_counts()
transaction.commit()
print 'done.'
print 'Transferring comments...',
sys.stdout.flush()
self.transfer_comments()
transaction.commit()
print 'done.'
print 'Transferring badges and badge awards...',
sys.stdout.flush()
self.transfer_badges()
transaction.commit()
print 'done.'
print 'Transferring Q&A votes...',
sys.stdout.flush()
self.transfer_QA_votes()#includes favorites, accepts and flags
transaction.commit()
print 'done.'
print 'Transferring comment votes...',
sys.stdout.flush()
self.transfer_comment_votes()
transaction.commit()
self.cleanup_messages()#delete autogenerated messages
transaction.commit()
self.transfer_messages()
transaction.commit()
#todo: these are not clear how to go about
self.transfer_update_subscriptions()
transaction.commit()
self.transfer_tag_preferences()
transaction.commit()
self.transfer_meta_pages()
transaction.commit()
print 'done.'
def handle(self, *arg, **kwarg):
if django_settings.DEBUG:
raise CommandError(
'Please set DEBUG to False in the settings.py to reduce '
'RAM usage during the import process'
)
#process the command line arguments, if given
if kwarg['read_dump'] is False and kwarg['process_data'] is False:
#make them both true as a hack to simulate a condition where
#no flags selected means the same as both are indeed selected
kwarg['read_dump'] = True
kwarg['process_data'] = True
askbot_settings.update('LIMIT_ONE_ANSWER_PER_USER', False)
if not importer_is_ready():
raise CommandError(
"Looks like stackexchange tables are not yet initialized in the database.\n"
"Please, run command: \npython manage.py syncdb\n"
"then import the data."
)
award_badges_signal.disconnect(award_badges)
if len(arg) < 1 or not os.path.isfile(arg[0]):
raise CommandError('Error: first argument must be a zip file with the SE forum data')
if kwarg['read_dump']:
self.zipfile = self.open_dump(arg[0])
#read the data into SE tables
for item in xml_read_order:
time_before = datetime.now()
self.load_xml_file(item)
transaction.commit()
time_after = datetime.now()
if DEBUGME == True:
print time_after - time_before
print HEAP.heap()
if kwarg['process_data'] is False:
#that means we just wanted to load the xml dump to
#do the second step in another go in order to have
#more ram for the transfer of data from SE to Askbot databases
return
#this is important so that when we clean up messages
#automatically generated by the procedures below
#we do not delete old messages
#todo: unfortunately this may need to be redone
#when we upgrade to django 1.2 and definitely by 1.4 when
#the current message system will be replaced with the
#django messages framework
self.save_askbot_message_id_list()
#transfer data into ASKBOT tables
print 'Transferring users...'
self.transfer_users()
transaction.commit()
print 'done.'
print 'Transferring content edits...'
sys.stdout.flush()
self.transfer_question_and_answer_activity()
transaction.commit()
print 'done.'
print 'Transferring view counts...'
sys.stdout.flush()
self.transfer_question_view_counts()
transaction.commit()
print 'done.'
print 'Transferring comments...'
sys.stdout.flush()
self.transfer_comments()
transaction.commit()
print 'done.'
print 'Transferring badges and badge awards...'
sys.stdout.flush()
self.transfer_badges()
transaction.commit()
print 'done.'
print 'Transferring Q&A votes...'
sys.stdout.flush()
self.transfer_QA_votes()#includes favorites, accepts and flags
transaction.commit()
print 'done.'
print 'Transferring comment votes...'
sys.stdout.flush()
self.transfer_comment_votes()
transaction.commit()
self.cleanup_messages()#delete autogenerated messages
transaction.commit()
self.transfer_messages()
transaction.commit()
#todo: these are not clear how to go about
self.transfer_update_subscriptions()
transaction.commit()
self.transfer_tag_preferences()
transaction.commit()
self.transfer_meta_pages()
transaction.commit()
print 'done.'
