def gen_profile(): logger.info(f"Transferring profiles") # Exclude existing users from source database. users = UsersUser.objects.all().order_by("id") elapsed, progress = timer_func() # Allow limiting the input stream = islice(zip(count(1), users), limit) for index, user in stream: progress(index, msg="profiles") text = util.strip_tags(user.profile.info) profile = Profile(uid=user.id, user=current.get(user.email), name=user.name, role=user.type, last_login=user.last_login, html=user.profile.info, date_joined=user.profile.date_joined, location=user.profile.location, website=user.profile.website, scholar=user.profile.scholar, text=text, score=user.score, twitter=user.profile.twitter_id, my_tags=user.profile.my_tags, digest_prefs=user.profile.digest_prefs, new_messages=user.new_messages) yield profile
def gen_profile(): logger.info(f"Transferring profiles") elapsed, progress = timer_func() # Allow limiting the input stream = islice(zip(count(1), old_users), limit) for index, user in stream: progress(index, msg="profiles") text = util.strip_tags(user.profile.info) # The incoming users have weekly digest prefs as a default. #digest_prefs = Profile.NO_DIGEST if user.profile.digest_prefs == Profile.WEEKLY_DIGEST else user.profile.digest_prefs profile = Profile(uid=user.id, user=current.get(user.email), name=user.name, message_prefs=user.profile.message_prefs, role=user.type, last_login=user.last_login, html=user.profile.info, date_joined=user.profile.date_joined, location=user.profile.location, website=user.profile.website, scholar=user.profile.scholar, text=text, score=user.score, twitter=user.profile.twitter_id, my_tags=user.profile.my_tags, digest_prefs=user.profile.digest_prefs, new_messages=user.new_messages) yield profile
def create_blogpost(entry, blog): date = entry.get('date_parsed') or entry.get('published_parsed') date = datetime(date[0], date[1], date[2]) date = timezone.make_aware(date, timezone=timezone.utc) if not entry.title: return # body = html.clean(entry.description)[:5000] body = entry.description content = strip_tags(body) try: post = BlogPost.objects.create(title=entry.title, blog=blog, uid=entry.id, content=content, html=body, creation_date=date, link=entry.link) except Exception as exc: logger.error(entry.title) logger.error(f"database error {exc}") else: logger.info(f"added: {post.title}") return
def gen_posts(): logger.info("transferring posts") posts = PostsPost.objects.order_by("id")[:LIMIT] users_set = {user.profile.uid: user for user in all_users} elapsed, progress = timer_func() stream = zip(count(1), posts) stream = islice(stream, LIMIT) for index, post in stream: progress(index, msg="posts") author = users_set.get(str(post.author_id)) lastedit_user = users_set.get(str(post.lastedit_user_id)) # Incomplete author information loaded or existing posts. if not (author and lastedit_user): continue rank = post.lastedit_date.timestamp() content = util.strip_tags(post.content) new_post = Post(uid=post.id, html=post.html, type=post.type, lastedit_user=lastedit_user, thread_votecount=post.thread_score, author=author, status=post.status, rank=rank, has_accepted=post.has_accepted, lastedit_date=post.lastedit_date, book_count=post.book_count, reply_count=post.reply_count, content=content, title=post.title, vote_count=post.vote_count, thread_score=post.reply_count, creation_date=post.creation_date, tag_val=post.tag_val, view_count=post.view_count) # Store parent and root for every post. relations[str(new_post.uid)] = [str(post.root_id), str(post.parent_id)] yield new_post
def sync_users(users): """ Sync users one at a time instead of bulk creating. """ # Get the column names column = users.get('column') rows = users.get('rows', []) added = dict() # Check if this user already exists. for row in rows: # Map column names to row. row = {col: val for col, val in zip(column, row)} # Skip if user exists. user = User.objects.filter(email=row['email']).first() if user: added[row['user_id']] = user continue # Create the user username = f"{row['name'].replace(' ', '-')}-{row['user_id']}" user = User.objects.create(username=username, email=row['email'], password=row['password'], is_active=row['is_active'], is_staff=row['is_staff'], is_superuser=row['is_admin']) text = util.strip_tags(row['info']) # Update the Profile Profile.objects.filter(user=user).update( digest_prefs=row['digest_prefs'], watched_tags=row['watched_tags'], twitter=row['twitter_id'], uid=row['user_id'], name=row['name'], message_prefs=row['message_prefs'], role=row['type'], last_login=row['last_login'], html=row['info'], date_joined=row['date_joined'], location=row['location'], website=row['website'], scholar=row['scholar'], text=text, score=row['score'], my_tags=row['my_tags'], new_messages=row['new_messages']) added[row['user_id']] = user logger.info(f"Updated {len(rows)} users.") return added
def gen_posts(): logger.info("transferring posts") posts = PostsPost.objects.order_by("id") elapsed, progress = timer_func() stream = zip(count(1), posts) stream = islice(stream, limit) for index, post in stream: progress(index, msg="posts") author = users_set.get(str(post.author_id)) lastedit_user = users_set.get(str(post.lastedit_user_id)) # Incomplete author information loaded or existing posts. if not (author and lastedit_user): continue siblings = posts.filter(root_id=post.root_id) # Record replies, comments, and answers to root reply_count = siblings.count() comment_count = siblings.filter(type=Post.COMMENT).count() rank = post.lastedit_date.timestamp() content = util.strip_tags(post.content) new_post = Post(uid=post.id, html=post.html, type=post.type, reply_count=reply_count, lastedit_user=lastedit_user, thread_votecount=post.thread_score, author=author, status=post.status, rank=rank, accept_count=int(post.has_accepted), lastedit_date=post.lastedit_date, book_count=post.book_count, comment_count=comment_count, content=content, title=post.title, vote_count=post.vote_count, creation_date=post.creation_date, tag_val=post.tag_val, answer_count=post.reply_count, view_count=post.view_count) # Store parent and root for every post. relations[str( new_post.uid)] = [str(post.root_id), str(post.parent_id)] yield new_post
def save(self, *args, **kwargs): # Sanitize the post body. self.html = util.parse_html(self.content) # Must add tags with instance method. This is just for safety. self.tag_val = util.strip_tags(self.tag_val) # Posts other than a question also carry the same tag if self.is_toplevel and self.type != Post.QUESTION: required_tag = self.get_type_display() if required_tag not in self.tag_val: self.tag_val += "," + required_tag # Recompute post reply count self.update_reply_count() super(Post, self).save(*args, **kwargs)
def save(self, author=None, edit=False): data = self.cleaned_data.get title = data('title') html = data("content") content = util.strip_tags(html) post_type = int(data('post_type')) tag_val = data('tag_val') if edit: self.post.title = title self.post.content = content self.post.type = post_type self.post.html = html self.post.save() # Triggers another save self.post.add_tags(text=tag_val) else: self.post = auth.create_post(title=title, content=content, post_type=post_type, tag_val=tag_val, author=author, project=self.project) return self.post
def save(self, author=None, post_type=Post.ANSWER, edit=False): data = self.cleaned_data html = data.get("content") project = data.get("project_uid") parent = data.get("parent_uid") content = util.strip_tags(html) if edit: self.post.html = html self.post.content = content self.post.save() else: parent = Post.objects.filter(uid=parent).first() project = Project.objects.filter(uid=project).first() self.post = auth.create_post(title=parent.root.title, parent=parent, author=author, content=content, post_type=post_type, project=project, sub_to_root=True, ) return self.post