def run(cls):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
cls.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts()
# debug mode: no sync, just stream
if Conf.get('disable_sync'):
return cls.listen()
while True:
# sync up to irreversible block
cls.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# listen for new blocks
cls.listen()
def run_sync():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
Sync.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
CachedPost.recover_missing_posts()
while True:
# sync up to irreversible block
Sync.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
Sync.listen()
def run():
if db_needs_setup():
print("[INIT] Initializing db...")
setup()
#TODO: if initial sync is interrupted, cache never rebuilt
#TODO: do not build partial feed_cache during init_sync
# if this is the initial sync, batch updates until very end
is_initial_sync = not query_one("SELECT 1 FROM hive_posts_cache LIMIT 1")
if is_initial_sync:
print("[INIT] *** Initial sync. db_last_block: %d ***" %
db_last_block())
else:
# perform cleanup in case process did not exit cleanly
cache_missing_posts()
# prefetch id->name memory map
Accounts.load_ids()
# fast block sync strategies
sync_from_checkpoints(is_initial_sync)
sync_from_steemd(is_initial_sync)
Accounts.cache_old()
Accounts.update_ranks()
if is_initial_sync:
print("[INIT] *** Initial sync complete. Rebuilding cache. ***")
cache_missing_posts()
rebuild_feed_cache()
# initialization complete. follow head blocks
listen_steemd()
def run(self):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
Accounts.fetch_ranks()
Community.recalc_pending_payouts()
if DbState.is_initial_sync():
# resume initial sync
self.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head(self._steem)
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts(self._steem)
#audit_cache_missing(self._db, self._steem)
#audit_cache_deleted(self._db)
self._update_chain_state()
if self._conf.get('test_max_block'):
# debug mode: partial sync
return self.from_steemd()
if self._conf.get('test_disable_sync'):
# debug mode: no sync, just stream
return self.listen()
while True:
# sync up to irreversible block
self.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(self._steem, trx=True)
try:
# listen for new blocks
self.listen()
except MicroForkException as e:
# attempt to recover by restarting stream
log.error("NOTIFYALERT microfork: %s", repr(e))
def _check_migrations(cls):
"""Check current migration version and perform updates as needed."""
cls._ver = cls.db().query_one(
"SELECT db_version FROM hive_state LIMIT 1")
assert cls._ver is not None, 'could not load state record'
if cls._ver == 0:
raise Exception("dbv cannot be 0; reindex required")
if cls._ver == 1:
cls._set_ver(2)
if cls._ver == 2:
cls._set_ver(3)
if cls._ver == 3:
sql = """CREATE INDEX hive_accounts_ix3 ON hive_accounts
USING btree (vote_weight, name varchar_pattern_ops)"""
cls.db().query(sql)
cls._set_ver(4)
if cls._ver == 4:
sql = """CREATE INDEX hive_follows_ix4 ON public.hive_follows
USING btree (follower, following) WHERE state = 2;"""
cls.db().query(sql)
cls._set_ver(5)
if cls._ver == 5:
# recover acct names lost to issue #151
from hive.steem.client import SteemClient
from hive.indexer.accounts import Accounts
names = SteemClient().get_all_account_names()
Accounts.load_ids()
Accounts.register(names, '1970-01-01T00:00:00')
Accounts.clear_ids()
cls._set_ver(6)
assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"
def run():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
print("[INIT] *** Initial fast sync ***")
sync_from_checkpoints()
sync_from_steemd()
print("[INIT] *** Initial cache build ***")
# todo: disable indexes during this process
cache_missing_posts()
FeedCache.rebuild()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup in case process did not exit cleanly
cache_missing_posts()
while True:
# sync up to irreversible block
sync_from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# start listening
listen_steemd()
def _check_migrations(cls):
"""Check current migration version and perform updates as needed."""
#pylint: disable=line-too-long
cls._ver = cls.db().query_one(
"SELECT db_version FROM hive_state LIMIT 1")
assert cls._ver is not None, 'could not load state record'
if cls._ver == 0:
raise Exception("dbv cannot be 0; reindex required")
if cls._ver == 1:
cls._set_ver(2)
if cls._ver == 2:
cls._set_ver(3)
if cls._ver == 3:
cls.db().query(
"CREATE INDEX hive_accounts_ix3 ON hive_accounts (vote_weight, name varchar_pattern_ops)"
)
cls._set_ver(4)
if cls._ver == 4:
cls.db().query(
"CREATE INDEX hive_follows_ix4 ON hive_follows (follower, following) WHERE state = 2"
)
cls._set_ver(5)
if cls._ver == 5:
# recover acct names lost to issue #151
from hive.steem.client import SteemClient
from hive.indexer.accounts import Accounts
names = SteemClient().get_all_account_names()
Accounts.load_ids()
Accounts.register(names, '1970-01-01T00:00:00')
Accounts.clear_ids()
cls._set_ver(6)
if cls._ver == 6:
cls.db().query("DROP INDEX hive_posts_cache_ix6")
cls.db().query(
"CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'"
)
cls.db().query("DROP INDEX hive_posts_cache_ix7")
cls.db().query(
"CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'"
)
cls._set_ver(7)
if cls._ver == 7:
cls.db().query(
"CREATE INDEX hive_accounts_ix4 ON hive_accounts (id, name)")
cls.db().query(
"CREATE INDEX hive_accounts_ix5 ON hive_accounts (cached_at, name)"
)
cls._set_ver(8)
if cls._ver == 8:
cls.db().query("DROP INDEX hive_follows_ix2")
cls.db().query("DROP INDEX hive_follows_ix3")
cls.db().query("DROP INDEX hive_follows_ix4")
cls.db().query(
"CREATE INDEX hive_follows_5a ON hive_follows (following, state, created_at, follower)"
)
cls.db().query(
"CREATE INDEX hive_follows_5b ON hive_follows (follower, state, created_at, following)"
)
cls._set_ver(9)
if cls._ver == 9:
from hive.indexer.follow import Follow
Follow.force_recount()
cls._set_ver(10)
if cls._ver == 10:
cls.db().query(
"CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'"
)
cls._set_ver(11)
if cls._ver == 11:
cls.db().query("DROP INDEX hive_posts_ix1")
cls.db().query("DROP INDEX hive_posts_ix2")
cls.db().query(
"CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE is_deleted = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE is_deleted = '0'"
)
cls._set_ver(12)
reset_autovac(cls.db())
log.info("[HIVE] db version: %d", cls._ver)
assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"
def run(self):
old_sig_int_handler = getsignal(SIGINT)
old_sig_term_handler = getsignal(SIGTERM)
set_handlers()
Community.start_block = self._conf.get("community_start_block")
# ensure db schema up to date, check app status
DbState.initialize()
if self._conf.get("log_explain_queries"):
is_superuser = self._db.query_one("SELECT is_superuser()")
assert is_superuser, 'The parameter --log_explain_queries=true can be used only when connect to the database with SUPERUSER privileges'
_is_consistency = Blocks.is_consistency()
if not _is_consistency:
raise RuntimeError(
"Fatal error related to `hive_blocks` consistency")
show_info(self._db)
paths = self._conf.get("mock_block_data_path") or []
for path in paths:
self.load_mock_data(path)
mock_vops_data_path = self._conf.get("mock_vops_data_path")
if mock_vops_data_path:
MockVopsProvider.load_block_data(mock_vops_data_path)
# MockVopsProvider.print_data()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
# community stats
update_communities_posts_and_rank(self._db)
last_imported_block = Blocks.head_num()
hived_head_block = self._conf.get(
'test_max_block') or self._steem.last_irreversible()
log.info("target_head_block : %s", hived_head_block)
if DbState.is_initial_sync():
DbState.before_initial_sync(last_imported_block, hived_head_block)
# resume initial sync
self.initial()
if not can_continue_thread():
restore_handlers()
return
current_imported_block = Blocks.head_num()
# beacuse we cannot break long sql operations, then we back default CTRL+C
# behavior for the time of post initial actions
restore_handlers()
try:
DbState.finish_initial_sync(current_imported_block)
except KeyboardInterrupt:
log.info("Break finish initial sync")
set_exception_thrown()
return
set_handlers()
else:
# recover from fork
Blocks.verify_head(self._steem)
self._update_chain_state()
global trail_blocks
trail_blocks = self._conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
def _check_migrations(cls):
"""Check current migration version and perform updates as needed."""
#pylint: disable=line-too-long,too-many-branches,too-many-statements
cls._ver = cls.db().query_one(
"SELECT db_version FROM hive_state LIMIT 1")
assert cls._ver is not None, 'could not load state record'
if cls._ver == 0:
raise Exception("dbv cannot be 0; reindex required")
if cls._ver == 1:
cls._set_ver(2)
if cls._ver == 2:
cls._set_ver(3)
if cls._ver == 3:
cls.db().query(
"CREATE INDEX hive_accounts_ix3 ON hive_accounts (vote_weight, name varchar_pattern_ops)"
)
cls._set_ver(4)
if cls._ver == 4:
cls.db().query(
"CREATE INDEX hive_follows_ix4 ON hive_follows (follower, following) WHERE state = 2"
)
cls._set_ver(5)
if cls._ver == 5:
# recover acct names lost to issue #151
from hive.steem.client import SteemClient
from hive.indexer.accounts import Accounts
names = SteemClient().get_all_account_names()
Accounts.load_ids()
Accounts.register(names, '1970-01-01T00:00:00')
Accounts.clear_ids()
cls._set_ver(6)
if cls._ver == 6:
cls.db().query("DROP INDEX hive_posts_cache_ix6")
cls.db().query(
"CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'"
)
cls.db().query("DROP INDEX hive_posts_cache_ix7")
cls.db().query(
"CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'"
)
cls._set_ver(7)
if cls._ver == 7:
cls.db().query(
"CREATE INDEX hive_accounts_ix4 ON hive_accounts (id, name)")
cls.db().query(
"CREATE INDEX hive_accounts_ix5 ON hive_accounts (cached_at, name)"
)
cls._set_ver(8)
if cls._ver == 8:
cls.db().query("DROP INDEX hive_follows_ix2")
cls.db().query("DROP INDEX hive_follows_ix3")
cls.db().query("DROP INDEX hive_follows_ix4")
cls.db().query(
"CREATE INDEX hive_follows_5a ON hive_follows (following, state, created_at, follower)"
)
cls.db().query(
"CREATE INDEX hive_follows_5b ON hive_follows (follower, state, created_at, following)"
)
cls._set_ver(9)
if cls._ver == 9:
from hive.indexer.follow import Follow
Follow.force_recount()
cls._set_ver(10)
if cls._ver == 10:
cls.db().query(
"CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'"
)
cls._set_ver(11)
if cls._ver == 11:
cls.db().query("DROP INDEX hive_posts_ix1")
cls.db().query("DROP INDEX hive_posts_ix2")
cls.db().query(
"CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE is_deleted = '0'"
)
cls.db().query(
"CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE is_deleted = '0'"
)
cls._set_ver(12)
if cls._ver == 12: # community schema
assert False, 'not finalized'
for table in [
'hive_members', 'hive_flags', 'hive_modlog',
'hive_communities', 'hive_subscriptions', 'hive_roles',
'hive_notifs'
]:
cls.db().query("DROP TABLE IF EXISTS %s" % table)
build_metadata_community().create_all(cls.db().engine())
cls.db().query(
"ALTER TABLE hive_accounts ADD COLUMN lr_notif_id integer")
cls.db().query(
"ALTER TABLE hive_posts DROP CONSTRAINT hive_posts_fk2")
cls.db().query("ALTER TABLE hive_posts DROP COLUMN community")
cls.db().query(
"ALTER TABLE hive_posts ADD COLUMN community_id integer")
cls.db().query(
"ALTER TABLE hive_posts_cache ADD COLUMN community_id integer")
cls._set_ver(13)
if cls._ver == 13:
sqls = (
"CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'",
"CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'",
"CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0",
"CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'",
"CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'"
)
for sql in sqls:
cls.db().query(sql)
cls._set_ver(14)
if cls._ver == 14:
cls.db().query(
"ALTER TABLE hive_communities ADD COLUMN primary_tag VARCHAR(32) NOT NULL DEFAULT ''"
)
cls.db().query(
"ALTER TABLE hive_communities ADD COLUMN category VARCHAR(32) NOT NULL DEFAULT ''"
)
cls.db().query(
"ALTER TABLE hive_communities ADD COLUMN avatar_url VARCHAR(1024) NOT NULL DEFAULT ''"
)
cls.db().query(
"ALTER TABLE hive_communities ADD COLUMN num_authors INTEGER NOT NULL DEFAULT 0"
)
cls.db().query(
"CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'"
)
cls._set_ver(15)
if cls._ver == 15:
cls.db().query("ALTER TABLE hive_accounts DROP COLUMN lr_notif_id")
cls.db().query(
"ALTER TABLE hive_accounts ADD COLUMN lastread_at TIMESTAMP WITHOUT TIME ZONE DEFAULT '1970-01-01 00:00:00' NOT NULL"
)
cls.db().query(
"CREATE INDEX hive_notifs_ix6 ON hive_notifs (dst_id, created_at, score, id) WHERE dst_id IS NOT NULL"
)
cls._set_ver(16)
reset_autovac(cls.db())
log.info("[HIVE] db version: %d", cls._ver)
assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"