def steem(self): """Get a SteemClient instance, lazily initialized""" if not self._steem: self._steem = SteemClient(url=self.get('steemd_url'), max_batch=self.get('max_batch'), max_workers=self.get('max_workers')) return self._steem
def steem(self): """Get a SteemClient instance, lazily initialized""" if not self._steem: from json import loads self._steem = SteemClient( url=loads(self.get('steemd_url')), max_batch=self.get('max_batch'), max_workers=self.get('max_workers'), max_retries=self.get('max_retries')) return self._steem
def _check_migrations(cls): """Check current migration version and perform updates as needed.""" cls._ver = cls.db().query_one( "SELECT db_version FROM hive_state LIMIT 1") assert cls._ver is not None, 'could not load state record' if cls._ver == 0: raise Exception("dbv cannot be 0; reindex required") if cls._ver == 1: cls._set_ver(2) if cls._ver == 2: cls._set_ver(3) if cls._ver == 3: sql = """CREATE INDEX hive_accounts_ix3 ON hive_accounts USING btree (vote_weight, name varchar_pattern_ops)""" cls.db().query(sql) cls._set_ver(4) if cls._ver == 4: sql = """CREATE INDEX hive_follows_ix4 ON public.hive_follows USING btree (follower, following) WHERE state = 2;""" cls.db().query(sql) cls._set_ver(5) if cls._ver == 5: # recover acct names lost to issue #151 from hive.steem.client import SteemClient from hive.indexer.accounts import Accounts names = SteemClient().get_all_account_names() Accounts.load_ids() Accounts.register(names, '1970-01-01T00:00:00') Accounts.clear_ids() cls._set_ver(6) assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"
def client(): return SteemClient(url='https://api.steemit.com')
def _check_migrations(cls): """Check current migration version and perform updates as needed.""" #pylint: disable=line-too-long cls._ver = cls.db().query_one( "SELECT db_version FROM hive_state LIMIT 1") assert cls._ver is not None, 'could not load state record' if cls._ver == 0: raise Exception("dbv cannot be 0; reindex required") if cls._ver == 1: cls._set_ver(2) if cls._ver == 2: cls._set_ver(3) if cls._ver == 3: cls.db().query( "CREATE INDEX hive_accounts_ix3 ON hive_accounts (vote_weight, name varchar_pattern_ops)" ) cls._set_ver(4) if cls._ver == 4: cls.db().query( "CREATE INDEX hive_follows_ix4 ON hive_follows (follower, following) WHERE state = 2" ) cls._set_ver(5) if cls._ver == 5: # recover acct names lost to issue #151 from hive.steem.client import SteemClient from hive.indexer.accounts import Accounts names = SteemClient().get_all_account_names() Accounts.load_ids() Accounts.register(names, '1970-01-01T00:00:00') Accounts.clear_ids() cls._set_ver(6) if cls._ver == 6: cls.db().query("DROP INDEX hive_posts_cache_ix6") cls.db().query( "CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'" ) cls.db().query("DROP INDEX hive_posts_cache_ix7") cls.db().query( "CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'" ) cls._set_ver(7) if cls._ver == 7: cls.db().query( "CREATE INDEX hive_accounts_ix4 ON hive_accounts (id, name)") cls.db().query( "CREATE INDEX hive_accounts_ix5 ON hive_accounts (cached_at, name)" ) cls._set_ver(8) if cls._ver == 8: cls.db().query("DROP INDEX hive_follows_ix2") cls.db().query("DROP INDEX hive_follows_ix3") cls.db().query("DROP INDEX hive_follows_ix4") cls.db().query( "CREATE INDEX hive_follows_5a ON hive_follows (following, state, created_at, follower)" ) cls.db().query( "CREATE INDEX hive_follows_5b ON hive_follows (follower, state, created_at, following)" ) cls._set_ver(9) if cls._ver == 9: from hive.indexer.follow import Follow Follow.force_recount() cls._set_ver(10) if cls._ver == 10: cls.db().query( "CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'" ) cls._set_ver(11) if cls._ver == 11: cls.db().query("DROP INDEX hive_posts_ix1") cls.db().query("DROP INDEX hive_posts_ix2") cls.db().query( "CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE is_deleted = '0'" ) cls.db().query( "CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE is_deleted = '0'" ) cls._set_ver(12) reset_autovac(cls.db()) log.info("[HIVE] db version: %d", cls._ver) assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"
from hive.steem.client import SteemClient if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument("hived_url", type=str, help="Url address of hived instance") parser.add_argument("from_block", type=int, help="Scan from block") parser.add_argument("to_block", type=int, help="Scan to block") parser.add_argument("output_file", type=str, help="Prepared blocks will be saved in this file") parser.add_argument("operations", type=str, nargs='+', help="Save selected operations") parser.add_argument("--dump-ops-only", type=bool, default=False, help="Dump only selected ops, without block data") args = parser.parse_args() client = SteemClient({"default":args.hived_url}) from_block = args.from_block def breaker(): return True with open(args.output_file, "w") as output_file: if not args.dump_ops_only: output_file.write("{\n") while from_block < args.to_block: to_block = from_block + 1000 if to_block >= args.to_block: to_block = args.to_block + 1 print("Processing range from: ", from_block, " to: ", to_block) blocks = client.get_blocks_range(from_block, to_block, breaker) for block in blocks:
def client(): return SteemClient(url='https://api.hive.blog')
from hive.server.condenser_api.objects import (load_accounts, load_posts) from hive.server.condenser_api.common import (valid_account, valid_permlink, valid_sort, valid_tag, get_post_id, get_child_ids) from hive.server.condenser_api.methods import (get_replies_by_last_update, get_discussions_by_comments, get_discussions_by_blog, get_discussions_by_feed) from hive.server.condenser_api.tags import (get_trending_tags, get_top_trending_tags_summary) import hive.server.condenser_api.cursor as cursor log = logging.getLogger(__name__) TMP_STEEM = SteemClient() # steemd account 'tabs' - specific post list queries ACCOUNT_TAB_KEYS = { '': 'blog', 'feed': 'feed', 'comments': 'comments', 'recent-replies': 'recent_replies' } # dummy account paths used by condenser - just need account object ACCOUNT_TAB_IGNORE = [ 'followed', 'followers', 'permissions', 'password', 'settings' ] # misc dummy paths used by condenser - send minimal get_state structure
def _check_migrations(cls): """Check current migration version and perform updates as needed.""" #pylint: disable=line-too-long,too-many-branches,too-many-statements cls._ver = cls.db().query_one( "SELECT db_version FROM hive_state LIMIT 1") assert cls._ver is not None, 'could not load state record' if cls._ver == 0: raise Exception("dbv cannot be 0; reindex required") if cls._ver == 1: cls._set_ver(2) if cls._ver == 2: cls._set_ver(3) if cls._ver == 3: cls.db().query( "CREATE INDEX hive_accounts_ix3 ON hive_accounts (vote_weight, name varchar_pattern_ops)" ) cls._set_ver(4) if cls._ver == 4: cls.db().query( "CREATE INDEX hive_follows_ix4 ON hive_follows (follower, following) WHERE state = 2" ) cls._set_ver(5) if cls._ver == 5: # recover acct names lost to issue #151 from hive.steem.client import SteemClient from hive.indexer.accounts import Accounts names = SteemClient().get_all_account_names() Accounts.load_ids() Accounts.register(names, '1970-01-01T00:00:00') Accounts.clear_ids() cls._set_ver(6) if cls._ver == 6: cls.db().query("DROP INDEX hive_posts_cache_ix6") cls.db().query( "CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'" ) cls.db().query("DROP INDEX hive_posts_cache_ix7") cls.db().query( "CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'" ) cls._set_ver(7) if cls._ver == 7: cls.db().query( "CREATE INDEX hive_accounts_ix4 ON hive_accounts (id, name)") cls.db().query( "CREATE INDEX hive_accounts_ix5 ON hive_accounts (cached_at, name)" ) cls._set_ver(8) if cls._ver == 8: cls.db().query("DROP INDEX hive_follows_ix2") cls.db().query("DROP INDEX hive_follows_ix3") cls.db().query("DROP INDEX hive_follows_ix4") cls.db().query( "CREATE INDEX hive_follows_5a ON hive_follows (following, state, created_at, follower)" ) cls.db().query( "CREATE INDEX hive_follows_5b ON hive_follows (follower, state, created_at, following)" ) cls._set_ver(9) if cls._ver == 9: from hive.indexer.follow import Follow Follow.force_recount() cls._set_ver(10) if cls._ver == 10: cls.db().query( "CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'" ) cls._set_ver(11) if cls._ver == 11: cls.db().query("DROP INDEX hive_posts_ix1") cls.db().query("DROP INDEX hive_posts_ix2") cls.db().query( "CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE is_deleted = '0'" ) cls.db().query( "CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE is_deleted = '0'" ) cls._set_ver(12) if cls._ver == 12: # community schema assert False, 'not finalized' for table in [ 'hive_members', 'hive_flags', 'hive_modlog', 'hive_communities', 'hive_subscriptions', 'hive_roles', 'hive_notifs' ]: cls.db().query("DROP TABLE IF EXISTS %s" % table) build_metadata_community().create_all(cls.db().engine()) cls.db().query( "ALTER TABLE hive_accounts ADD COLUMN lr_notif_id integer") cls.db().query( "ALTER TABLE hive_posts DROP CONSTRAINT hive_posts_fk2") cls.db().query("ALTER TABLE hive_posts DROP COLUMN community") cls.db().query( "ALTER TABLE hive_posts ADD COLUMN community_id integer") cls.db().query( "ALTER TABLE hive_posts_cache ADD COLUMN community_id integer") cls._set_ver(13) if cls._ver == 13: sqls = ( "CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'", "CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'", "CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0", "CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", "CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", "CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", "CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'", "CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'" ) for sql in sqls: cls.db().query(sql) cls._set_ver(14) if cls._ver == 14: cls.db().query( "ALTER TABLE hive_communities ADD COLUMN primary_tag VARCHAR(32) NOT NULL DEFAULT ''" ) cls.db().query( "ALTER TABLE hive_communities ADD COLUMN category VARCHAR(32) NOT NULL DEFAULT ''" ) cls.db().query( "ALTER TABLE hive_communities ADD COLUMN avatar_url VARCHAR(1024) NOT NULL DEFAULT ''" ) cls.db().query( "ALTER TABLE hive_communities ADD COLUMN num_authors INTEGER NOT NULL DEFAULT 0" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'" ) cls._set_ver(15) if cls._ver == 15: cls.db().query("ALTER TABLE hive_accounts DROP COLUMN lr_notif_id") cls.db().query( "ALTER TABLE hive_accounts ADD COLUMN lastread_at TIMESTAMP WITHOUT TIME ZONE DEFAULT '1970-01-01 00:00:00' NOT NULL" ) cls.db().query( "CREATE INDEX hive_notifs_ix6 ON hive_notifs (dst_id, created_at, score, id) WHERE dst_id IS NOT NULL" ) cls._set_ver(16) reset_autovac(cls.db()) log.info("[HIVE] db version: %d", cls._ver) assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"