def reblog(cls, account, op_json, block_date):
"""Handle legacy 'reblog' op"""
blogger = op_json['account']
author = op_json['author']
permlink = op_json['permlink']
if blogger != account:
return # impersonation
if not all(map(Accounts.exists, [author, blogger])):
return
post_id, depth = Posts.get_id_and_depth(author, permlink)
if depth > 0:
return # prevent comment reblogs
if not post_id:
log.debug("reblog: post not found: %s/%s", author, permlink)
return
if 'delete' in op_json and op_json['delete'] == 'delete':
DB.query("DELETE FROM hive_reblogs WHERE account = :a AND "
"post_id = :pid LIMIT 1", a=blogger, pid=post_id)
if not DbState.is_initial_sync():
FeedCache.delete(post_id, Accounts.get_id(blogger))
else:
sql = ("INSERT INTO hive_reblogs (account, post_id, created_at) "
"VALUES (:a, :pid, :date) ON CONFLICT (account, post_id) DO NOTHING")
DB.query(sql, a=blogger, pid=post_id, date=block_date)
if not DbState.is_initial_sync():
FeedCache.insert(post_id, Accounts.get_id(blogger), block_date)
def run(cls):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
cls.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts()
# debug mode: no sync, just stream
if Conf.get('disable_sync'):
return cls.listen()
while True:
# sync up to irreversible block
cls.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# listen for new blocks
cls.listen()
def run_sync():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
Sync.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
CachedPost.recover_missing_posts()
while True:
# sync up to irreversible block
Sync.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
Sync.listen()
def run(self):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
Accounts.fetch_ranks()
Community.recalc_pending_payouts()
if DbState.is_initial_sync():
# resume initial sync
self.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head(self._steem)
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts(self._steem)
#audit_cache_missing(self._db, self._steem)
#audit_cache_deleted(self._db)
self._update_chain_state()
if self._conf.get('test_max_block'):
# debug mode: partial sync
return self.from_steemd()
if self._conf.get('test_disable_sync'):
# debug mode: no sync, just stream
return self.listen()
while True:
# sync up to irreversible block
self.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(self._steem, trx=True)
try:
# listen for new blocks
self.listen()
except MicroForkException as e:
# attempt to recover by restarting stream
log.error("NOTIFYALERT microfork: %s", repr(e))
def insert(cls, post_id, account_id, created_at):
"""Inserts a [re-]post by an account into feed."""
assert not DbState.is_initial_sync(), 'writing to feed cache in sync'
sql = """INSERT INTO hive_feed_cache (account_id, post_id, created_at)
VALUES (:account_id, :id, :created_at)
ON CONFLICT (account_id, post_id) DO NOTHING"""
query(sql, account_id=account_id, id=post_id, created_at=created_at)
def op_transfer(cls, op, tx_idx, num, date):
"""Process raw transfer op; apply balance if valid post promote."""
NativeAd.check_ad_payment(op, date, num)
record = cls._validated(op, tx_idx, num, date)
if not record:
return
# add payment record
sql = DB.build_insert('hive_payments', record, pk='id')
DB.query(sql)
# read current amount
sql = "SELECT promoted FROM hive_posts WHERE id = :id"
curr_amount = DB.query_one(sql, id=record['post_id'])
new_amount = curr_amount + record['amount']
# update post record
sql = "UPDATE hive_posts SET promoted = :val WHERE id = :id"
DB.query(sql, val=new_amount, id=record['post_id'])
# notify cached_post of new promoted balance, and trigger update
if not DbState.is_initial_sync():
CachedPost.update_promoted_amount(record['post_id'], new_amount)
author, permlink = cls._split_url(op['memo'])
CachedPost.vote(author, permlink, record['post_id'])
def follow_op(cls, account, op_json, date):
"""Process an incoming follow op."""
op = cls._validated_op(account, op_json, date)
if not op:
return
# perform delta check
new_state = op['state']
old_state = cls._get_follow_db_state(op['flr'], op['flg'])
if new_state == (old_state or 0):
return
# insert or update state
if old_state is None:
sql = """INSERT INTO hive_follows (follower, following,
created_at, state) VALUES (:flr, :flg, :at, :state)"""
else:
sql = """UPDATE hive_follows SET state = :state
WHERE follower = :flr AND following = :flg"""
DB.query(sql, **op)
# track count deltas
if not DbState.is_initial_sync():
if new_state == 1:
Follow.follow(op['flr'], op['flg'])
if old_state == 1:
Follow.unfollow(op['flr'], op['flg'])
def run():
"""Run the proper routine as indicated by hive --mode argument."""
conf = Conf.init_argparse()
Db.set_shared_instance(conf.db())
mode = conf.mode()
if mode == 'server':
from hive.server.serve import run_server
run_server(conf=conf)
elif mode == 'sync':
from hive.indexer.sync import Sync
Sync(conf=conf).run()
elif mode == 'status':
from hive.db.db_state import DbState
print(DbState.status())
#elif mode == 'sync-profile':
# from hive.indexer.sync import Sync
# from hive.utils.profiler import Profiler
# with Profiler():
# Sync(conf=conf).run()
else:
raise Exception("unknown run mode %s" % mode)
def insert(cls, op, date):
"""Inserts new post records."""
sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author,
permlink, category, community_id, depth, created_at)
VALUES (:is_valid, :is_muted, :parent_id, :author,
:permlink, :category, :community_id, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author'] + '/' + op['permlink'], post['id'])
if not DbState.is_initial_sync():
if post['error']:
author_id = Accounts.get_id(post['author'])
Notify('error',
dst_id=author_id,
when=date,
post_id=post['id'],
payload=post['error']).write()
CachedPost.insert(op['author'], op['permlink'], post['id'])
if op['parent_author']: # update parent's child count
CachedPost.recount(op['parent_author'], op['parent_permlink'],
post['parent_id'])
cls._insert_feed_cache(post)
def initial(self):
"""Initial sync routine."""
assert DbState.is_initial_sync(), "already synced"
log.info("[INIT] *** Initial fast sync ***")
self.from_steemd(is_initial_sync=True)
if not can_continue_thread():
return
def delete(cls, op):
pid, depth = cls.get_id_and_depth(op['author'], op['permlink'])
query("UPDATE hive_posts SET is_deleted = '1' WHERE id = :id", id=pid)
if not DbState.is_initial_sync():
CachedPost.delete(pid, op['author'], op['permlink'])
if depth == 0:
FeedCache.delete(pid)
def update(cls, op, date, pid):
"""Handle post updates.
Here we could also build content diffs, but for now just used
a signal to update cache record.
"""
# pylint: disable=unused-argument
if not DbState.is_initial_sync():
CachedPost.update(op['author'], op['permlink'], pid)
def undelete(cls, op, date, pid):
sql = """UPDATE hive_posts SET is_valid = :is_valid, is_deleted = '0',
parent_id = :parent_id, category = :category,
community = :community, depth = :depth
WHERE id = :id"""
post = cls._build_post(op, date, pid)
DB.query(sql, **post)
if not DbState.is_initial_sync():
CachedPost.undelete(pid, post['author'], post['permlink'])
cls._insert_feed_cache(post)
def initial(cls):
assert DbState.is_initial_sync(), "already synced"
print("[INIT] *** Initial fast sync ***")
cls.from_checkpoints()
cls.from_steemd(is_initial_sync=True)
print("[INIT] *** Initial cache build ***")
# TODO: disable indexes during this process
CachedPost.recover_missing_posts()
FeedCache.rebuild()
def initial(self):
"""Initial sync routine."""
assert DbState.is_initial_sync(), "already synced"
log.info("[INIT] *** Initial fast sync ***")
self.from_checkpoints()
self.from_steemd(is_initial_sync=True)
log.info("[INIT] *** Initial cache build ***")
CachedPost.recover_missing_posts(self._steem)
FeedCache.rebuild()
Follow.force_recount()
def delete(cls, post_id, account_id=None):
"""Remove a post from feed cache.
If `account_id` is specified, we remove a single entry (e.g. a
singular un-reblog). Otherwise, we remove all instances of the
post (e.g. a post was deleted; its entry and all reblogs need
to be removed.
"""
assert not DbState.is_initial_sync(), 'writing to feed cache in sync'
sql = "DELETE FROM hive_feed_cache WHERE post_id = :id"
if account_id:
sql = sql + " AND account_id = :account_id"
query(sql, account_id=account_id, id=post_id)
def sync_from_steemd():
is_initial_sync = DbState.is_initial_sync()
steemd = get_adapter()
lbound = Blocks.head_num() + 1
ubound = steemd.last_irreversible()
if ubound <= lbound:
return
_abort = False
try:
print("[SYNC] start block %d, +%d to sync" % (lbound, ubound-lbound+1))
timer = Timer(ubound - lbound, entity='block', laps=['rps', 'wps'])
while lbound < ubound:
to = min(lbound + 1000, ubound)
timer.batch_start()
blocks = steemd.get_blocks_range(lbound, to)
timer.batch_lap()
Blocks.process_multi(blocks, is_initial_sync)
timer.batch_finish(len(blocks))
date = blocks[-1]['timestamp']
print(timer.batch_status("[SYNC] Got block %d @ %s" % (to-1, date)))
lbound = to
except KeyboardInterrupt:
traceback.print_exc()
print("\n\n[SYNC] Aborted.. cleaning up..")
_abort = True
if not is_initial_sync:
# Follows flushing may need to be moved closer to core (i.e. moved
# into main block transactions). Important to keep in sync since
# we need to prevent expensive recounts. This will fail if we aborted
# in the middle of a transaction, meaning data loss. Better than
# forcing it, however, since in-memory cache will be out of sync
# with db state.
Follow.flush(trx=True)
# This flush is low importance; accounts are swept regularly.
if not _abort:
Accounts.flush(trx=True)
# If this flush fails, all that could potentially be lost here is
# edits and pre-payout votes. If the post has not been paid out yet,
# then the worst case is it will be synced upon payout. If the post
# is already paid out, worst case is to lose an edit.
CachedPost.flush(trx=True)
if _abort:
print("[SYNC] Aborted")
exit()
def run():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
print("[INIT] *** Initial fast sync ***")
sync_from_checkpoints()
sync_from_steemd()
print("[INIT] *** Initial cache build ***")
# todo: disable indexes during this process
cache_missing_posts()
FeedCache.rebuild()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
cache_missing_posts()
while True:
# sync up to irreversible block
sync_from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
listen_steemd()
def delete(cls, op):
"""Marks a post record as being deleted."""
pid, depth = cls.get_id_and_depth(op['author'], op['permlink'])
DB.query("UPDATE hive_posts SET is_deleted = '1' WHERE id = :id", id=pid)
if not DbState.is_initial_sync():
CachedPost.delete(pid, op['author'], op['permlink'])
if depth == 0:
# TODO: delete from hive_reblogs -- otherwise feed cache gets populated with deleted posts somwrimas
FeedCache.delete(pid)
else:
# force parent child recount when child is deleted
prnt = cls._get_parent_by_child_id(pid)
CachedPost.recount(prnt['author'], prnt['permlink'], prnt['id'])
def insert(cls, op, date):
sql = """INSERT INTO hive_posts (is_valid, parent_id, author, permlink,
category, community, depth, created_at)
VALUES (:is_valid, :parent_id, :author, :permlink,
:category, :community, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author'] + '/' + op['permlink'], post['id'])
if not DbState.is_initial_sync():
CachedPost.insert(op['author'], op['permlink'], post['id'])
cls._insert_feed_cache(post)
def run():
Conf.init_argparse()
mode = '/'.join(Conf.get('mode'))
if mode == 'server':
run_server()
elif mode == 'sync':
run_sync()
elif mode == 'status':
print(DbState.status())
else:
raise Exception("unknown run mode %s" % mode)
def launch_mode(mode, conf):
"""Launch a routine as indicated by `mode`."""
if mode == 'server':
from hive.server.serve import run_server
run_server(conf=conf)
elif mode == 'sync':
from hive.indexer.sync import Sync
Sync(conf=conf).run()
elif mode == 'status':
from hive.db.db_state import DbState
print(DbState.status())
else:
raise Exception("unknown run mode %s" % mode)
def run():
"""Run the proper routine as indicated by hive --mode argument."""
Conf.init_argparse()
mode = Conf.run_mode()
if mode == 'server':
run_server()
elif mode == 'sync':
Sync.run()
elif mode == 'status':
print(DbState.status())
else:
raise Exception("unknown run mode %s" % mode)
def undelete(cls, op, date, pid):
"""Re-allocates an existing record flagged as deleted."""
sql = """UPDATE hive_posts SET is_valid = :is_valid,
is_muted = :is_muted, is_deleted = '0', is_pinned = '0',
parent_id = :parent_id, category = :category,
community_id = :community_id, depth = :depth
WHERE id = :id"""
post = cls._build_post(op, date, pid)
DB.query(sql, **post)
if not DbState.is_initial_sync():
if post['error']:
author_id = Accounts.get_id(post['author'])
Notify('error', dst_id=author_id, when=date,
post_id=post['id'], payload=post['error']).write()
CachedPost.undelete(pid, post['author'], post['permlink'],
post['category'])
cls._insert_feed_cache(post)
def run():
"""Run the proper routine as indicated by hive --mode argument."""
conf = Conf.init_argparse()
Db.set_shared_instance(conf.db())
mode = '/'.join(conf.get('mode'))
if mode == 'server':
from hive.server.serve import run_server
run_server(conf=conf)
elif mode == 'sync':
from hive.indexer.sync import Sync
Sync(conf=conf).run()
elif mode == 'status':
from hive.db.db_state import DbState
print(DbState.status())
else:
raise Exception("unknown run mode %s" % mode)
def _notify(self, op, **kwargs):
if DbState.is_initial_sync():
# TODO: set start date for notifs?
# TODO: address other callers
return
dst_id = None
score = 35
if self.account_id and not self.post_id:
dst_id = self.account_id
if not self._subscribed(self.account_id):
score = 15
Notify(op,
src_id=self.actor_id,
dst_id=dst_id,
post_id=self.post_id,
when=self.date,
community_id=self.community_id,
score=score,
**kwargs).write()
def process(self):
"""Applies a validated operation."""
assert self.valid, 'cannot apply invalid op'
from hive.indexer.cached_post import CachedPost
action = self.action
params = dict(
date=self.date,
community=self.community,
community_id=self.community_id,
actor=self.actor,
actor_id=self.actor_id,
account=self.account,
account_id=self.account_id,
post_id=self.post_id,
role_id=self.role_id,
notes=self.notes,
title=self.title,
)
# Community-level commands
if action == 'updateProps':
bind = ', '.join([k + " = :" + k for k in list(self.props.keys())])
DB.query("UPDATE hive_communities SET %s WHERE id = :id" % bind,
id=self.community_id,
**self.props)
self._notify('set_props',
payload=json.dumps(read_key_dict(self.op, 'props')))
elif action == 'subscribe':
DB.query(
"""INSERT INTO hive_subscriptions
(account_id, community_id, created_at)
VALUES (:actor_id, :community_id, :date)""", **params)
DB.query(
"""UPDATE hive_communities
SET subscribers = subscribers + 1
WHERE id = :community_id""", **params)
self._notify('subscribe')
elif action == 'unsubscribe':
DB.query(
"""DELETE FROM hive_subscriptions
WHERE account_id = :actor_id
AND community_id = :community_id""", **params)
DB.query(
"""UPDATE hive_communities
SET subscribers = subscribers - 1
WHERE id = :community_id""", **params)
# Account-level actions
elif action == 'setRole':
DB.query(
"""INSERT INTO hive_roles
(account_id, community_id, role_id, created_at)
VALUES (:account_id, :community_id, :role_id, :date)
ON CONFLICT (account_id, community_id)
DO UPDATE SET role_id = :role_id""", **params)
self._notify('set_role', payload=Role(self.role_id).name)
elif action == 'setUserTitle':
DB.query(
"""INSERT INTO hive_roles
(account_id, community_id, title, created_at)
VALUES (:account_id, :community_id, :title, :date)
ON CONFLICT (account_id, community_id)
DO UPDATE SET title = :title""", **params)
self._notify('set_label', payload=self.title)
# Post-level actions
elif action == 'mutePost':
DB.query(
"""UPDATE hive_posts SET is_muted = '1'
WHERE id = :post_id""", **params)
self._notify('mute_post', payload=self.notes)
if not DbState.is_initial_sync():
CachedPost.update(self.account, self.permlink, self.post_id)
elif action == 'unmutePost':
DB.query(
"""UPDATE hive_posts SET is_muted = '0'
WHERE id = :post_id""", **params)
self._notify('unmute_post', payload=self.notes)
if not DbState.is_initial_sync():
CachedPost.update(self.account, self.permlink, self.post_id)
elif action == 'pinPost':
DB.query(
"""UPDATE hive_posts SET is_pinned = '1'
WHERE id = :post_id""", **params)
self._notify('pin_post', payload=self.notes)
elif action == 'unpinPost':
DB.query(
"""UPDATE hive_posts SET is_pinned = '0'
WHERE id = :post_id""", **params)
self._notify('unpin_post', payload=self.notes)
elif action == 'flagPost':
self._notify('flag_post', payload=self.notes)
return True
async def db_head_state():
return DbState.status()
def delete(cls, post_id, account_id=None):
assert not DbState.is_initial_sync(), 'writing to feed cache in sync'
sql = "DELETE FROM hive_feed_cache WHERE post_id = :id"
if account_id:
sql = sql + " AND account_id = :account_id"
query(sql, account_id=account_id, id=post_id)
def run(self):
old_sig_int_handler = getsignal(SIGINT)
old_sig_term_handler = getsignal(SIGTERM)
set_handlers()
Community.start_block = self._conf.get("community_start_block")
# ensure db schema up to date, check app status
DbState.initialize()
if self._conf.get("log_explain_queries"):
is_superuser = self._db.query_one("SELECT is_superuser()")
assert is_superuser, 'The parameter --log_explain_queries=true can be used only when connect to the database with SUPERUSER privileges'
_is_consistency = Blocks.is_consistency()
if not _is_consistency:
raise RuntimeError(
"Fatal error related to `hive_blocks` consistency")
show_info(self._db)
paths = self._conf.get("mock_block_data_path") or []
for path in paths:
self.load_mock_data(path)
mock_vops_data_path = self._conf.get("mock_vops_data_path")
if mock_vops_data_path:
MockVopsProvider.load_block_data(mock_vops_data_path)
# MockVopsProvider.print_data()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
# community stats
update_communities_posts_and_rank(self._db)
last_imported_block = Blocks.head_num()
hived_head_block = self._conf.get(
'test_max_block') or self._steem.last_irreversible()
log.info("target_head_block : %s", hived_head_block)
if DbState.is_initial_sync():
DbState.before_initial_sync(last_imported_block, hived_head_block)
# resume initial sync
self.initial()
if not can_continue_thread():
restore_handlers()
return
current_imported_block = Blocks.head_num()
# beacuse we cannot break long sql operations, then we back default CTRL+C
# behavior for the time of post initial actions
restore_handlers()
try:
DbState.finish_initial_sync(current_imported_block)
except KeyboardInterrupt:
log.info("Break finish initial sync")
set_exception_thrown()
return
set_handlers()
else:
# recover from fork
Blocks.verify_head(self._steem)
self._update_chain_state()
global trail_blocks
trail_blocks = self._conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
