def do_render_cache(cursor=None): thq = Thread.all() if cursor: thq.with_cursor(cursor) thread = thq.get() if not thread: logging.info("stop thread clean") return board = thread.parent_key().name() render = Render(board=board, thread = thread.key().id()) for idx,post in enumerate(thread.posts): post['text_html'] = markup( board=board, postid=post.get("post"), data=escape(post.get('text', '')), ) if idx == 0: render.create(post) else: render.append(post) if len(thread.posts) > 1: thread.put() else: thread.delete() render.save() deferred.defer(do_render_cache, thq.cursor())
def get(self): list, list_url = get_list_from_url(self.request.get('list')) msg = None if list is None: render(self, 'error.html', msg = 'The specified list does not exist.') return # If the last time the list was updated was over 12 hours ago, schedule an # update if list.last_fetched_time < datetime.now() - timedelta(hours=12): schedule_list_update(list) msg = 'Fetching new messages...' threads = Thread.all().filter('list_url =', list_url).\ order('-last_message_time') formatted_threads = [] for t in threads: if len(t.participants) > 3: t.participants = [t.participants[0], '...'] + t.participants[-2:] t.participants = [', '.join(p.split()[0] for p in t.participants)] t.subject = strip_tags(t.subject) t.last_message_body = strip_tags(t.last_message_body)[:100] one_day = timedelta(days=1) if t.last_message_time > datetime.now() - timedelta(days=1): t.short_date = datetime.strftime(t.last_message_time, '%H:%M%p') elif t.last_message_time > datetime.now() - timedelta(days=365): t.short_date = datetime.strftime(t.last_message_time, '%b %d') else: t.short_date = datetime.strftime(t.last_message_time, '%D') t.read = False formatted_threads.append(t) user = users.get_current_user() if user: def lookup(): if len(lookup_pool) == 0: return for ut in UserThread.all().filter('thread_id IN', lookup_pool): last_viewed[ut.thread_id] = ut.last_viewed del lookup_pool[:] last_viewed = {} # maps from thread_ids to last viewed lookup_pool = [] for t in formatted_threads: lookup_pool.append(t.thread_id) if len(lookup_pool) == 30: lookup() lookup() for t in formatted_threads: if t.thread_id in last_viewed: if t.last_message_time <= last_viewed[t.thread_id]: t.read = True render(self, 'view.html', list = list, threads = formatted_threads, msg = msg)
def do_render_cache(cursor=None): thq = Thread.all(keys_only=True) if cursor: thq.with_cursor(cursor) if not thq.count(1): return for thread in thq.fetch(100): deferred.defer(process, thread, _queue='render') deferred.defer(do_render_cache, thq.cursor() )
def do_render_cache(cursor=None): thq = Thread.all() if cursor: thq.with_cursor(cursor) thread = thq.get() if not thread: logging.info("stop thread clean") return board = thread.board render = Render(board=board, thread = thread.id) for idx,post in enumerate(thread.posts): if 'text' in post: post['text_html'] = markup( board=board, postid=post.get("post"), data=escape(post.get('text', '')), ) if 'rainbow_html' in post: post.pop("rainbow_html") post['rainbow'] = rainbow.make_rainbow(post['rainbow']) if 'image' in post and not post.get("key"): post.pop("image") post['name'] = 'Kuroneko' if idx == 0: render.create(post) else: render.append(post) if len(thread.posts) > 1: thread.put() else: thread.delete() render.save() deferred.defer(do_render_cache, thq.cursor())
def get_post(board, num): key = "post-%(board)s-%(num)d" % {"board":board, "num":num} post = None #memcache.get(key) if post != None: logging.info("cache hit") return post thq = Thread.all() thq.ancestor( db.Key.from_path("Board", board)) thq.filter("post_numbers", num) thread = thq.get() if not thread: return [post] = [p for p in thread.posts if p.get('post') == num] memcache.set(key, post) return post
# Create thread labels print('Create thread labels...') if not ThreadLabel.count(): labels = set() while len(labels) != THREAD_LABELS_COUNT: labels.add(' '.join( fake.words(nb=random.randint(0, 3), ext_word_list=None))) for label in labels: ThreadLabel.create(text=label) labels = ThreadLabel.all() # TODO(a.telishev): Use cursor iterator threads = Thread.all() for i, thread in enumerate(threads): for label in random.choices(labels, k=random.randint( 0, MAX_LABELS_FOR_ONE_THREAD)): ThreadsLabels.create(thread=thread.id, label=label.id) if i and (i % 20000 == 0): print(f' - Processed {i} threads') # Create answers print('Create answers...') current_answers_count = Answer.count() if current_answers_count < ANSWERS_COUNT: threads = Thread.all() users = User.all()
def __init__(self, forum): self.threads = [] threads = Thread.all().filter('forum =',forum).order('-datetime').fetch(10,offset=0) for thread in threads: self.threads.append(ThreadViewModel(thread))