def gen_profile():
logger.info(f"Transferring profiles")
# Exclude existing users from source database.
users = UsersUser.objects.all().order_by("id")
elapsed, progress = timer_func()
# Allow limiting the input
stream = islice(zip(count(1), users), limit)
for index, user in stream:
progress(index, msg="profiles")
text = util.strip_tags(user.profile.info)
profile = Profile(uid=user.id,
user=current.get(user.email),
name=user.name,
role=user.type,
last_login=user.last_login,
html=user.profile.info,
date_joined=user.profile.date_joined,
location=user.profile.location,
website=user.profile.website,
scholar=user.profile.scholar,
text=text,
score=user.score,
twitter=user.profile.twitter_id,
my_tags=user.profile.my_tags,
digest_prefs=user.profile.digest_prefs,
new_messages=user.new_messages)
yield profile
def gen_profile():
logger.info(f"Transferring profiles")
elapsed, progress = timer_func()
# Allow limiting the input
stream = islice(zip(count(1), old_users), limit)
for index, user in stream:
progress(index, msg="profiles")
text = util.strip_tags(user.profile.info)
# The incoming users have weekly digest prefs as a default.
#digest_prefs = Profile.NO_DIGEST if user.profile.digest_prefs == Profile.WEEKLY_DIGEST else user.profile.digest_prefs
profile = Profile(uid=user.id,
user=current.get(user.email),
name=user.name,
message_prefs=user.profile.message_prefs,
role=user.type,
last_login=user.last_login,
html=user.profile.info,
date_joined=user.profile.date_joined,
location=user.profile.location,
website=user.profile.website,
scholar=user.profile.scholar,
text=text,
score=user.score,
twitter=user.profile.twitter_id,
my_tags=user.profile.my_tags,
digest_prefs=user.profile.digest_prefs,
new_messages=user.new_messages)
yield profile
def create_blogpost(entry, blog):
date = entry.get('date_parsed') or entry.get('published_parsed')
date = datetime(date[0], date[1], date[2])
date = timezone.make_aware(date, timezone=timezone.utc)
if not entry.title:
return
# body = html.clean(entry.description)[:5000]
body = entry.description
content = strip_tags(body)
try:
post = BlogPost.objects.create(title=entry.title,
blog=blog,
uid=entry.id,
content=content,
html=body,
creation_date=date,
link=entry.link)
except Exception as exc:
logger.error(entry.title)
logger.error(f"database error {exc}")
else:
logger.info(f"added: {post.title}")
return
def gen_posts():
logger.info("transferring posts")
posts = PostsPost.objects.order_by("id")[:LIMIT]
users_set = {user.profile.uid: user for user in all_users}
elapsed, progress = timer_func()
stream = zip(count(1), posts)
stream = islice(stream, LIMIT)
for index, post in stream:
progress(index, msg="posts")
author = users_set.get(str(post.author_id))
lastedit_user = users_set.get(str(post.lastedit_user_id))
# Incomplete author information loaded or existing posts.
if not (author and lastedit_user):
continue
rank = post.lastedit_date.timestamp()
content = util.strip_tags(post.content)
new_post = Post(uid=post.id, html=post.html, type=post.type,
lastedit_user=lastedit_user, thread_votecount=post.thread_score,
author=author, status=post.status, rank=rank, has_accepted=post.has_accepted,
lastedit_date=post.lastedit_date, book_count=post.book_count, reply_count=post.reply_count,
content=content, title=post.title, vote_count=post.vote_count,
thread_score=post.reply_count,
creation_date=post.creation_date, tag_val=post.tag_val,
view_count=post.view_count)
# Store parent and root for every post.
relations[str(new_post.uid)] = [str(post.root_id), str(post.parent_id)]
yield new_post
def sync_users(users):
"""
Sync users one at a time instead of bulk creating.
"""
# Get the column names
column = users.get('column')
rows = users.get('rows', [])
added = dict()
# Check if this user already exists.
for row in rows:
# Map column names to row.
row = {col: val for col, val in zip(column, row)}
# Skip if user exists.
user = User.objects.filter(email=row['email']).first()
if user:
added[row['user_id']] = user
continue
# Create the user
username = f"{row['name'].replace(' ', '-')}-{row['user_id']}"
user = User.objects.create(username=username,
email=row['email'],
password=row['password'],
is_active=row['is_active'],
is_staff=row['is_staff'],
is_superuser=row['is_admin'])
text = util.strip_tags(row['info'])
# Update the Profile
Profile.objects.filter(user=user).update(
digest_prefs=row['digest_prefs'],
watched_tags=row['watched_tags'],
twitter=row['twitter_id'],
uid=row['user_id'],
name=row['name'],
message_prefs=row['message_prefs'],
role=row['type'],
last_login=row['last_login'],
html=row['info'],
date_joined=row['date_joined'],
location=row['location'],
website=row['website'],
scholar=row['scholar'],
text=text,
score=row['score'],
my_tags=row['my_tags'],
new_messages=row['new_messages'])
added[row['user_id']] = user
logger.info(f"Updated {len(rows)} users.")
return added
def gen_posts():
logger.info("transferring posts")
posts = PostsPost.objects.order_by("id")
elapsed, progress = timer_func()
stream = zip(count(1), posts)
stream = islice(stream, limit)
for index, post in stream:
progress(index, msg="posts")
author = users_set.get(str(post.author_id))
lastedit_user = users_set.get(str(post.lastedit_user_id))
# Incomplete author information loaded or existing posts.
if not (author and lastedit_user):
continue
siblings = posts.filter(root_id=post.root_id)
# Record replies, comments, and answers to root
reply_count = siblings.count()
comment_count = siblings.filter(type=Post.COMMENT).count()
rank = post.lastedit_date.timestamp()
content = util.strip_tags(post.content)
new_post = Post(uid=post.id,
html=post.html,
type=post.type,
reply_count=reply_count,
lastedit_user=lastedit_user,
thread_votecount=post.thread_score,
author=author,
status=post.status,
rank=rank,
accept_count=int(post.has_accepted),
lastedit_date=post.lastedit_date,
book_count=post.book_count,
comment_count=comment_count,
content=content,
title=post.title,
vote_count=post.vote_count,
creation_date=post.creation_date,
tag_val=post.tag_val,
answer_count=post.reply_count,
view_count=post.view_count)
# Store parent and root for every post.
relations[str(
new_post.uid)] = [str(post.root_id),
str(post.parent_id)]
yield new_post
def save(self, *args, **kwargs):
# Sanitize the post body.
self.html = util.parse_html(self.content)
# Must add tags with instance method. This is just for safety.
self.tag_val = util.strip_tags(self.tag_val)
# Posts other than a question also carry the same tag
if self.is_toplevel and self.type != Post.QUESTION:
required_tag = self.get_type_display()
if required_tag not in self.tag_val:
self.tag_val += "," + required_tag
# Recompute post reply count
self.update_reply_count()
super(Post, self).save(*args, **kwargs)
def save(self, author=None, edit=False):
data = self.cleaned_data.get
title = data('title')
html = data("content")
content = util.strip_tags(html)
post_type = int(data('post_type'))
tag_val = data('tag_val')
if edit:
self.post.title = title
self.post.content = content
self.post.type = post_type
self.post.html = html
self.post.save()
# Triggers another save
self.post.add_tags(text=tag_val)
else:
self.post = auth.create_post(title=title, content=content, post_type=post_type,
tag_val=tag_val, author=author, project=self.project)
return self.post
def save(self, author=None, post_type=Post.ANSWER, edit=False):
data = self.cleaned_data
html = data.get("content")
project = data.get("project_uid")
parent = data.get("parent_uid")
content = util.strip_tags(html)
if edit:
self.post.html = html
self.post.content = content
self.post.save()
else:
parent = Post.objects.filter(uid=parent).first()
project = Project.objects.filter(uid=project).first()
self.post = auth.create_post(title=parent.root.title,
parent=parent,
author=author,
content=content,
post_type=post_type,
project=project,
sub_to_root=True,
)
return self.post
