def do_render_cache(cursor=None):
thq = Thread.all()
if cursor:
thq.with_cursor(cursor)
thread = thq.get()
if not thread:
logging.info("stop thread clean")
return
board = thread.parent_key().name()
render = Render(board=board, thread = thread.key().id())
for idx,post in enumerate(thread.posts):
post['text_html'] = markup(
board=board, postid=post.get("post"),
data=escape(post.get('text', '')),
)
if idx == 0:
render.create(post)
else:
render.append(post)
if len(thread.posts) > 1:
thread.put()
else:
thread.delete()
render.save()
deferred.defer(do_render_cache, thq.cursor())
def get(self):
list, list_url = get_list_from_url(self.request.get('list'))
msg = None
if list is None:
render(self, 'error.html', msg = 'The specified list does not exist.')
return
# If the last time the list was updated was over 12 hours ago, schedule an
# update
if list.last_fetched_time < datetime.now() - timedelta(hours=12):
schedule_list_update(list)
msg = 'Fetching new messages...'
threads = Thread.all().filter('list_url =', list_url).\
order('-last_message_time')
formatted_threads = []
for t in threads:
if len(t.participants) > 3:
t.participants = [t.participants[0], '...'] + t.participants[-2:]
t.participants = [', '.join(p.split()[0] for p in t.participants)]
t.subject = strip_tags(t.subject)
t.last_message_body = strip_tags(t.last_message_body)[:100]
one_day = timedelta(days=1)
if t.last_message_time > datetime.now() - timedelta(days=1):
t.short_date = datetime.strftime(t.last_message_time, '%H:%M%p')
elif t.last_message_time > datetime.now() - timedelta(days=365):
t.short_date = datetime.strftime(t.last_message_time, '%b %d')
else:
t.short_date = datetime.strftime(t.last_message_time, '%D')
t.read = False
formatted_threads.append(t)
user = users.get_current_user()
if user:
def lookup():
if len(lookup_pool) == 0:
return
for ut in UserThread.all().filter('thread_id IN', lookup_pool):
last_viewed[ut.thread_id] = ut.last_viewed
del lookup_pool[:]
last_viewed = {} # maps from thread_ids to last viewed
lookup_pool = []
for t in formatted_threads:
lookup_pool.append(t.thread_id)
if len(lookup_pool) == 30:
lookup()
lookup()
for t in formatted_threads:
if t.thread_id in last_viewed:
if t.last_message_time <= last_viewed[t.thread_id]:
t.read = True
render(self, 'view.html', list = list, threads = formatted_threads,
msg = msg)
def do_render_cache(cursor=None):
thq = Thread.all(keys_only=True)
if cursor:
thq.with_cursor(cursor)
if not thq.count(1):
return
for thread in thq.fetch(100):
deferred.defer(process, thread, _queue='render')
deferred.defer(do_render_cache, thq.cursor() )
def do_render_cache(cursor=None):
thq = Thread.all()
if cursor:
thq.with_cursor(cursor)
thread = thq.get()
if not thread:
logging.info("stop thread clean")
return
board = thread.board
render = Render(board=board, thread = thread.id)
for idx,post in enumerate(thread.posts):
if 'text' in post:
post['text_html'] = markup(
board=board, postid=post.get("post"),
data=escape(post.get('text', '')),
)
if 'rainbow_html' in post:
post.pop("rainbow_html")
post['rainbow'] = rainbow.make_rainbow(post['rainbow'])
if 'image' in post and not post.get("key"):
post.pop("image")
post['name'] = 'Kuroneko'
if idx == 0:
render.create(post)
else:
render.append(post)
if len(thread.posts) > 1:
thread.put()
else:
thread.delete()
render.save()
deferred.defer(do_render_cache, thq.cursor())
def get_post(board, num):
key = "post-%(board)s-%(num)d" % {"board":board, "num":num}
post = None #memcache.get(key)
if post != None:
logging.info("cache hit")
return post
thq = Thread.all()
thq.ancestor( db.Key.from_path("Board", board))
thq.filter("post_numbers", num)
thread = thq.get()
if not thread:
return
[post] = [p for p in thread.posts if p.get('post') == num]
memcache.set(key, post)
return post
# Create thread labels
print('Create thread labels...')
if not ThreadLabel.count():
labels = set()
while len(labels) != THREAD_LABELS_COUNT:
labels.add(' '.join(
fake.words(nb=random.randint(0, 3), ext_word_list=None)))
for label in labels:
ThreadLabel.create(text=label)
labels = ThreadLabel.all()
# TODO(a.telishev): Use cursor iterator
threads = Thread.all()
for i, thread in enumerate(threads):
for label in random.choices(labels,
k=random.randint(
0, MAX_LABELS_FOR_ONE_THREAD)):
ThreadsLabels.create(thread=thread.id, label=label.id)
if i and (i % 20000 == 0):
print(f' - Processed {i} threads')
# Create answers
print('Create answers...')
current_answers_count = Answer.count()
if current_answers_count < ANSWERS_COUNT:
threads = Thread.all()
users = User.all()
def __init__(self, forum):
self.threads = []
threads = Thread.all().filter('forum =',forum).order('-datetime').fetch(10,offset=0)
for thread in threads:
self.threads.append(ThreadViewModel(thread))
