def audit_cache_missing(db, steem):
"""Scan all posts to check for missing cache entries."""
last_id = _last_cached_post_id(db)
step = 1000000
steps = int(last_id / step) + 1
log.info("last post id: %d, batches: %d", last_id, steps)
sql = """
SELECT hp.id, hp.author, hp.permlink
FROM hive_posts hp
LEFT JOIN hive_posts_cache hpc
ON hp.id = hpc.post_id
WHERE hp.is_deleted = False
AND hp.id BETWEEN :lbound AND :ubound
AND hpc.post_id IS NULL"""
for idx in range(steps):
lbound = (idx * step) + 1
ubound = (idx + 1) * step
missing = db.query_all(sql, lbound=lbound, ubound=ubound)
log.info("%d <= id <= %d: %d missing", lbound, ubound, len(missing))
for row in missing:
CachedPost.insert(row['author'], row['permlink'], row['id'])
CachedPost.flush(steem, trx=True)
def insert(cls, op, date):
"""Inserts new post records."""
sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author,
permlink, category, community_id, depth, created_at)
VALUES (:is_valid, :is_muted, :parent_id, :author,
:permlink, :category, :community_id, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author'] + '/' + op['permlink'], post['id'])
if not DbState.is_initial_sync():
if post['error']:
author_id = Accounts.get_id(post['author'])
Notify('error',
dst_id=author_id,
when=date,
post_id=post['id'],
payload=post['error']).write()
CachedPost.insert(op['author'], op['permlink'], post['id'])
if op['parent_author']: # update parent's child count
CachedPost.recount(op['parent_author'], op['parent_permlink'],
post['parent_id'])
cls._insert_feed_cache(post)
def insert(cls, op, date):
sql = """INSERT INTO hive_posts (is_valid, parent_id, author, permlink,
category, community, depth, created_at)
VALUES (:is_valid, :parent_id, :author, :permlink,
:category, :community, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author'] + '/' + op['permlink'], post['id'])
if not DbState.is_initial_sync():
CachedPost.insert(op['author'], op['permlink'], post['id'])
cls._insert_feed_cache(post)