def migrate_posts(self, user_dict, topic_dict):
self.stdout.write("\n *** Migrate phpBB posts entries...\n")
posts = phpbb_Post.objects.all().order_by("time")
total = posts.count()
count = 0
start_time = time.time()
next_status = start_time + 0.25
for phpbb_post in posts:
count += 1
if time.time() > next_status:
self.stdout.write("\r\t%i/%i posts migrated... " % (count, total))
self.stdout.flush()
next_status = time.time() + 1
topic = topic_dict[phpbb_post.topic.id]
user = user_dict[phpbb_post.poster.id]
if phpbb_post.edit_user > 0 and phpbb_post.edit_time > 0:
updated = phpbb_post.update_datetime()
updated_by = user_dict[phpbb_post.edit_user]
else:
updated = None
updated_by = None
post = Post.objects.create(
topic=topic,
user=user,
created=phpbb_post.create_datetime(),
updated=updated,
updated_by=updated_by,
markup="bbcode",
body=phpbb_post.get_cleaned_bbcode(),
# body_html=html, # would be generated in save()
user_ip=phpbb_post.poster_ip,
)
if phpbb_post.has_attachment():
# copy attachment files
phpbb_attachment = phpbb_Attachment.objects.get(post_msg=phpbb_post)
src_path = os.path.join(settings.PHPBB_ATTACHMENT_PATH, phpbb_attachment.physical_filename)
if not os.path.isfile(src_path):
self._warn("\n +++ ERROR: Attachment not found: '%s'" % src_path)
else:
attachment = Attachment(
size=phpbb_attachment.filesize,
content_type=phpbb_attachment.mimetype,
name=phpbb_attachment.real_filename,
post=post,
)
filename = "%d.0" % post.id
dst_path = os.path.join(settings.MEDIA_ROOT, forum_settings.ATTACHMENT_UPLOAD_TO, filename)
shutil.copy(src_path, dst_path)
attachment.path = filename
attachment.save()
self.stdout.write("\n\t *** Attachment %s copied in: %s\n" % (attachment, dst_path))
self.stdout.flush()
duration = time.time() - start_time
self.stdout.write("\n *** %i posts migrated in %isec.\n" % (count, duration))
def save_attachment(self, post, memfile):
if memfile:
obj = Attachment(size=memfile.size, content_type=memfile.content_type,
name=memfile.name, post=post)
dir = os.path.join(settings.MEDIA_ROOT, forum_settings.ATTACHMENT_UPLOAD_TO)
fname = '%d.0' % post.id
path = os.path.join(dir, fname)
open(path, 'wb').write(memfile.read())
obj.path = fname
obj.save()
def save_attachment(self, post, memfile):
if memfile:
obj = Attachment(size=memfile.size, content_type=memfile.content_type,
name=memfile.name, post=post)
dir = os.path.join(settings.MEDIA_ROOT, forum_settings.ATTACHMENT_UPLOAD_TO)
fname = '%d.0' % post.id
path = os.path.join(dir, fname)
file(path, 'wb').write(memfile.read())
obj.path = fname
obj.save()
def save_attachment(self, post, memfile):
if memfile:
obj = Attachment(size=memfile.size, content_type=memfile.content_type,
name=memfile.name, post=post)
if forum_settings.ATTACHMENT_EC2_SUPPORT:
from djangobb_forum.attachment_storage import AttachmentStorage
attachment_storage = AttachmentStorage()
attachment_storage.add_attachment(post.id, memfile)
obj.path = post.id
else:
dir = os.path.join(settings.MEDIA_ROOT, forum_settings.ATTACHMENT_UPLOAD_TO)
fname = '%d.0' % post.id
path = os.path.join(dir, fname)
file(path, 'wb').write(memfile.read())
obj.path = fname
obj.save()
def save_attachment(self, post, memfile):
if memfile:
obj = Attachment(size=memfile.size,
content_type=memfile.content_type,
name=memfile.name,
post=post)
if forum_settings.ATTACHMENT_EC2_SUPPORT:
from djangobb_forum.attachment_storage import AttachmentStorage
attachment_storage = AttachmentStorage()
attachment_storage.add_attachment(post.id, memfile)
obj.path = post.id
else:
dir = os.path.join(settings.MEDIA_ROOT,
forum_settings.ATTACHMENT_UPLOAD_TO)
fname = '%d.0' % post.id
path = os.path.join(dir, fname)
file(path, 'wb').write(memfile.read())
obj.path = fname
obj.save()
def migrate_posts(self, user_dict):
self.out(u"\n *** Migrate phpBB posts entries...\n")
anonymous_user = self._get_anonymous_user(
) # Pseudo account from phpBB
posts = phpbb_Post.objects.all().order_by("time")
total = posts.count()
process_info = ProcessInfo(total, use_last_rates=4)
start_time = time.time()
next_status = start_time + 0.25
for count, phpbb_post in enumerate(posts, 1):
if self.max_entries and count >= self.max_entries:
self.warn("Skip rest of posts because max_entries used!")
break
if time.time() > next_status:
next_status = time.time() + 1
rest, eta, rate = process_info.update(count)
self.out_update(
"\t%i/%i posts migrated... rest: %i - eta: %s (rate: %.1f/sec)"
% (count, total, rest, eta, rate))
topic_id = phpbb_post.topic_id
try:
topic = Topic.objects.get(id=topic_id)
except Topic.DoesNotExist:
self.out_overwrite(
self.style.NOTICE(
"topic for post %i doesn't exist! Skip post." %
phpbb_post.id))
continue
phpbb_user_id = phpbb_post.poster_id
try:
user = user_dict[phpbb_user_id]
except KeyError:
self.out_overwrite(
self.style.NOTICE(
"phpBB User with ID %i doesn't exist for post %i. Use Anonymous."
% (phpbb_user_id, phpbb_post.id)))
user = anonymous_user
if phpbb_post.edit_user > 0 and phpbb_post.edit_time > 0:
updated = phpbb_post.update_datetime()
try:
updated_by = user_dict[phpbb_post.edit_user]
except KeyError:
updated_by = anonymous_user
else:
updated = None
updated_by = None
try:
post = Post.objects.create(
id=phpbb_post.id,
topic=topic,
user=user,
created=phpbb_post.create_datetime(),
updated=updated,
updated_by=updated_by,
markup="bbcode",
body=phpbb_post.get_cleaned_bbcode(),
#body_html=html, # would be generated in save()
user_ip=phpbb_post.poster_ip,
)
except Exception, err:
raise
msg = (
"\n +++ ERROR: creating Post entry for phpBB3 post (ID: %s):\n"
"%s\n") % (phpbb_post.id, err)
self.out_overwrite(self.style.NOTICE(msg))
continue
if phpbb_post.has_attachment():
# copy attachment files
phpbb_attachments = phpbb_Attachment.objects.filter(
post_msg=phpbb_post)
for phpbb_attachment in phpbb_attachments:
src_path = os.path.join(settings.PHPBB_ATTACHMENT_PATH,
phpbb_attachment.physical_filename)
if not os.path.isfile(src_path):
self.warn(
"\r\n +++ ERROR: Attachment not found: '%s'\n" %
src_path)
else:
attachment = Attachment(
size=phpbb_attachment.filesize,
content_type=phpbb_attachment.mimetype,
name=phpbb_attachment.real_filename,
post=post)
filename = "%d.0" % post.id
dst_path = os.path.join(
settings.MEDIA_ROOT,
forum_settings.ATTACHMENT_UPLOAD_TO, filename)
shutil.copy(src_path, dst_path)
attachment.path = filename
attachment.save()
self.out_overwrite(
"\t *** Attachment %s copied in: %s" %
(attachment.name, dst_path))
def migrate_posts(self, user_dict, topic_dict):
self.stdout.write("\n *** Migrate phpBB posts entries...\n")
posts = phpbb_Post.objects.all().order_by("time")
total = posts.count()
process_info = ProcessInfo(total, use_last_rates=4)
count = 0
start_time = time.time()
next_status = start_time + 0.25
for phpbb_post in posts:
count += 1
if time.time() > next_status:
next_status = time.time() + 1
rest, eta, rate = process_info.update(count)
msg = (
"\r\t%i/%i posts migrated... rest: %i - eta: %s (rate: %.1f/sec) "
) % (count, total, rest, eta, rate)
self.stdout.write(msg)
self.stdout.flush()
topic = topic_dict[phpbb_post.topic.id]
user = user_dict[phpbb_post.poster.id]
if phpbb_post.edit_user > 0 and phpbb_post.edit_time > 0:
updated = phpbb_post.update_datetime()
updated_by = user_dict[phpbb_post.edit_user]
else:
updated = None
updated_by = None
try:
post = Post.objects.create(
id=phpbb_post.id,
topic=topic,
user=user,
created=phpbb_post.create_datetime(),
updated=updated,
updated_by=updated_by,
markup="bbcode",
body=phpbb_post.get_cleaned_bbcode(),
#body_html=html, # would be generated in save()
user_ip=phpbb_post.poster_ip,
)
except Exception, err:
msg = (
"\n +++ ERROR: creating Post entry for phpBB3 post (ID: %s):\n"
"%s\n"
) % (phpbb_post.id, err)
self._warn(msg)
continue
if phpbb_post.has_attachment():
# copy attachment files
phpbb_attachment = phpbb_Attachment.objects.get(post_msg=phpbb_post)
src_path = os.path.join(settings.PHPBB_ATTACHMENT_PATH, phpbb_attachment.physical_filename)
if not os.path.isfile(src_path):
self._warn("\n +++ ERROR: Attachment not found: '%s'\n" % src_path)
else:
attachment = Attachment(
size=phpbb_attachment.filesize,
content_type=phpbb_attachment.mimetype,
name=phpbb_attachment.real_filename,
post=post
)
filename = "%d.0" % post.id
dst_path = os.path.join(
settings.MEDIA_ROOT, forum_settings.ATTACHMENT_UPLOAD_TO,
filename
)
shutil.copy(src_path, dst_path)
attachment.path = filename
attachment.save()
self.stdout.write(
"\n\t *** Attachment %s copied in: %s\n" % (
attachment, dst_path
)
)
self.stdout.flush()
def migrate_posts(self, user_dict):
self.out(u"\n *** Migrate phpBB posts entries...\n")
anonymous_user = self._get_anonymous_user() # Pseudo account from phpBB
posts = phpbb_Post.objects.all().order_by("time")
total = posts.count()
process_info = ProcessInfo(total, use_last_rates=4)
start_time = time.time()
next_status = start_time + 0.25
for count, phpbb_post in enumerate(posts, 1):
if self.max_entries and count >= self.max_entries:
self.warn("Skip rest of posts because max_entries used!")
break
if time.time() > next_status:
next_status = time.time() + 1
rest, eta, rate = process_info.update(count)
self.out_update(
"\t%i/%i posts migrated... rest: %i - eta: %s (rate: %.1f/sec)" % (
count, total, rest, eta, rate
)
)
topic_id = phpbb_post.topic_id
try:
topic = Topic.objects.get(id=topic_id)
except Topic.DoesNotExist:
self.out_overwrite(self.style.NOTICE(
"topic for post %i doesn't exist! Skip post." % phpbb_post.id
))
continue
phpbb_user_id = phpbb_post.poster_id
try:
user = user_dict[phpbb_user_id]
except KeyError:
self.out_overwrite(self.style.NOTICE(
"phpBB User with ID %i doesn't exist for post %i. Use Anonymous." % (
phpbb_user_id, phpbb_post.id
)
))
user = anonymous_user
if phpbb_post.edit_user > 0 and phpbb_post.edit_time > 0:
updated = phpbb_post.update_datetime()
try:
updated_by = user_dict[phpbb_post.edit_user]
except KeyError:
updated_by = anonymous_user
else:
updated = None
updated_by = None
try:
post = Post.objects.create(
id=phpbb_post.id,
topic=topic,
user=user,
created=phpbb_post.create_datetime(),
updated=updated,
updated_by=updated_by,
markup="bbcode",
body=phpbb_post.get_cleaned_bbcode(),
#body_html=html, # would be generated in save()
user_ip=phpbb_post.poster_ip,
)
except Exception, err:
raise
msg = (
"\n +++ ERROR: creating Post entry for phpBB3 post (ID: %s):\n"
"%s\n"
) % (phpbb_post.id, err)
self.out_overwrite(self.style.NOTICE(msg))
continue
if phpbb_post.has_attachment():
# copy attachment files
phpbb_attachments = phpbb_Attachment.objects.filter(post_msg=phpbb_post)
for phpbb_attachment in phpbb_attachments:
src_path = os.path.join(settings.PHPBB_ATTACHMENT_PATH, phpbb_attachment.physical_filename)
if not os.path.isfile(src_path):
self.warn("\r\n +++ ERROR: Attachment not found: '%s'\n" % src_path)
else:
attachment = Attachment(
size=phpbb_attachment.filesize,
content_type=phpbb_attachment.mimetype,
name=phpbb_attachment.real_filename,
post=post
)
filename = "%d.0" % post.id
dst_path = os.path.join(
settings.MEDIA_ROOT, forum_settings.ATTACHMENT_UPLOAD_TO,
filename
)
shutil.copy(src_path, dst_path)
attachment.path = filename
attachment.save()
self.out_overwrite(
"\t *** Attachment %s copied in: %s" % (
attachment.name, dst_path
)
)
