def run_steemd_tests( debug_node ): from steemapi.steemnoderpc import SteemNodeRPC try: print( 'Replaying blocks...', ) sys.stdout.flush() total_blocks = 0 while( total_blocks % 100000 == 0 ): total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True ) print( 'Blocks Replayed: ' + str( total_blocks ) ) sys.stdout.flush() print( "Triggering payouts" ) sys.stdout.flush() debug_node.debug_generate_blocks_until( 1467590400 ) print( "Generating blocks to verify nothing broke" ) assert( debug_node.debug_generate_blocks( 10 ) == 10 ) print( "Done!" ) print( "Getting comment dump:" ) sys.stdout.flush() rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' ) ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) ); print( 'author, url, total_payout_value, abs_rshares, num_active_votes' ) for comment in ret: print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] ) except ValueError as val_err: print( str( val_err ) )
def __enter__( self ): self._steemd_lock.acquire() # Setup temp directory to use as the data directory for this self._temp_data_dir = TemporaryDirectory() for child in self._data_dir.iterdir(): if( child.is_dir() ): copytree( str( child ), str( self._temp_data_dir.name ) + '/' + child.name ) db_version = Path( self._data_dir.name ) / 'db_version' if( db_version.exists() and not db_version.is_dir() ): copy2( str( db_version ), str( self._temp_data_dir.name ) + '/db_version' ) config = Path( self._temp_data_dir.name ) / 'config.ini' config.touch() config.write_text( self._get_config() ) steemd = [ str( self._steemd_bin ), '--data-dir=' + str( self._temp_data_dir.name ) ] steemd.extend( self._args ) self._steemd_process = Popen( steemd, stdout=self.steemd_out, stderr=self.steemd_err ) self._steemd_process.poll() sleep( 5 ) if( not self._steemd_process.returncode ): self._rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' ) else: raise Exception( "steemd did not start properly..." )
def run_steemd_tests( debug_node ): from steemapi.steemnoderpc import SteemNodeRPC try: print( "Playing blockchain..." ) blocks = debug_node.debug_push_blocks( 0 ) print( "Setting the hardfork now" ) # TODO: Grab most recent hardfork num from build directory sys.stdout.flush() debug_node.debug_set_hardfork( 4 ) print( "Generating blocks after the hardfork" ) assert( debug_node.debug_generate_blocks( 5000 ) == 5000 ) print( "Done!" ) print( "Calculating block producer distribution:" ) sys.stdout.flush() rpc = SteemNodeRPC( 'ws://127.0.0.1:8090', '', '' ) block_producers = {} for i in range( blocks + 1 , blocks + 5001 ): ret = rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [0,"get_block",[' + str( i ) + ']], "id":4}' ) ) if( ret[ "witness" ] in block_producers ): block_producers[ ret[ "witness" ] ] += 1 else: block_producers[ ret[ "witness" ] ] = 1 sorted_block_producers = sorted( block_producers.items(), key=operator.itemgetter( 1 ) ) for (k, v) in sorted_block_producers: ret = rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [0,"get_witness_by_account",["' + k + '"]], "id":5}' ) ) print( '{"witness":"' + k + '","votes":' + str( ret["votes"] ) + ',"blocks":' + str( v ) + '}' ) except ValueError as val_err: print( str( val_err ) )
def run_steemd_tests( debug_node ): from steemapi.steemnoderpc import SteemNodeRPC try: print( 'Replaying blocks...', ) sys.stdout.flush() total_blocks = 0 while( total_blocks % 100000 == 0 ): total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True ) print( 'Blocks Replayed: ' + str( total_blocks ) ) sys.stdout.flush() print( "Triggering payouts" ) sys.stdout.flush() debug_node.debug_generate_blocks_until( 1467590400 - 3 ) rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' ) ret = rpc.lookup_accounts( '', str( 0xFFFFFFFF ) ) account_balances = {} for acc_name in ret: acc = rpc.get_accounts( [ acc_name ] ) steemd = float( acc[0][ 'balance' ].split( ' ' )[0] ) sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] ) vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] ) account_balances[ acc_name ] = ( steemd, sbd, vests ) debug_node.debug_generate_blocks( 1 ) account_rewards = {} for acc_name, bal in account_balances.items(): acc = rpc.get_accounts( [ acc_name ] ) steemd = float( acc[0][ 'balance' ].split( ' ' )[0] ) - bal[0] sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] ) - bal[1] vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] ) - bal[2] account_rewards[ acc_name ] = ( steemd, sbd, vests ) print( "Generating blocks to verify nothing broke" ) assert( debug_node.debug_generate_blocks( 10 ) == 10 ) ''' print( "Done!" ) print( "Getting comment dump:" ) sys.stdout.flush() ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) ); print( 'author, url, total_payout_value, abs_rshares, num_active_votes' ) for comment in ret: print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] ) ''' print( "Printing account reward dump:" ) print( "account, steem, sbd, vests" ) for acc_name, rew in account_rewards.items(): print( acc_name + ', ' + str( rew[0] ) + ' STEEM, ' + str( rew[1] ) + ' SBD, ' + str( rew[2] ) + ' VESTS' ) except ValueError as val_err: print( str( val_err ) )
def run_steemd_tests(debug_node): from steemapi.steemnoderpc import SteemNodeRPC try: print('Replaying blocks...', ) sys.stdout.flush() total_blocks = 0 while (total_blocks % 100000 == 0): total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True) print('Blocks Replayed: ' + str(total_blocks)) sys.stdout.flush() print("Triggering payouts") sys.stdout.flush() debug_node.debug_generate_blocks_until(1467590400 - 3) rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '') ret = rpc.lookup_accounts('', str(0xFFFFFFFF)) debug_node.debug_generate_blocks(1) print("Generating blocks to verify nothing broke") assert (debug_node.debug_generate_blocks(10) == 10) account_rewards = {} vote_count = {} for acc_name in ret: acc = rpc.get_accounts([acc_name]) #print( acc_name + ',' + acc[0][ 'curation_rewards' ] ) account_rewards[acc_name] = float( acc[0]['curation_rewards'].split(' ')[0]) vote_count[acc_name] = int(acc[0]['lifetime_vote_count']) ''' print( "Done!" ) print( "Getting comment dump:" ) sys.stdout.flush() ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) ); print( 'author, url, total_payout_value, abs_rshares, num_active_votes' ) for comment in ret: print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] ) ''' print("Printing account reward dump:") sorted_rewards = sorted(account_rewards.items(), key=operator.itemgetter(1)) print("account, curation_steem") for rew in sorted_rewards: print(rew[0] + ', ' + str(rew[1]) + ', ' + str(vote_count[rew[0]])) except ValueError as val_err: print(str(val_err))
def run_steemd_tests( debug_node ): from steemapi.steemnoderpc import SteemNodeRPC try: print( 'Replaying blocks...', ) sys.stdout.flush() total_blocks = 0 while( total_blocks % 100000 == 0 ): total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True ) print( 'Blocks Replayed: ' + str( total_blocks ) ) sys.stdout.flush() print( "Triggering payouts" ) sys.stdout.flush() debug_node.debug_generate_blocks_until( 1467590400 - 3 ) rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' ) ret = rpc.lookup_accounts( '', str( 0xFFFFFFFF ) ) debug_node.debug_generate_blocks( 1 ) print( "Generating blocks to verify nothing broke" ) assert( debug_node.debug_generate_blocks( 10 ) == 10 ) account_rewards = {} vote_count = {} for acc_name in ret: acc = rpc.get_accounts( [ acc_name ] ) #print( acc_name + ',' + acc[0][ 'curation_rewards' ] ) account_rewards[ acc_name ] = float( acc[0][ 'curation_rewards' ].split( ' ' )[0] ) vote_count[ acc_name ] = int( acc[0][ 'lifetime_vote_count' ] ) ''' print( "Done!" ) print( "Getting comment dump:" ) sys.stdout.flush() ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) ); print( 'author, url, total_payout_value, abs_rshares, num_active_votes' ) for comment in ret: print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] ) ''' print( "Printing account reward dump:" ) sorted_rewards = sorted( account_rewards.items(), key=operator.itemgetter(1) ) print( "account, curation_steem" ) for rew in sorted_rewards: print( rew[0] + ', ' + str( rew[1] ) + ', ' + str( vote_count[ rew[0] ] ) ) except ValueError as val_err: print( str( val_err ) )
class FollowMe(object): def __init__(self): self.rpc = SteemNodeRPC("wss://node.steem.ws", "", "", apis=["follow"]) # get who we follow def following(self, account): return [ f['following'] for f in self.rpc.get_following( account, "", "blog", 100, api="follow") ]
def __enter__(self): self._steemd_lock.acquire() # Setup temp directory to use as the data directory for this self._temp_data_dir = TemporaryDirectory() config = Path(self._temp_data_dir.name) / 'config.ini' config.touch() config.write_text(self._get_config()) self._steemd_process = Popen([ str(self._steemd_bin), '--data-dir="' + str(self._temp_data_dir.name) + '"' ], stdout=self.steemd_out, stderr=self.steemd_err) self._steemd_process.poll() sleep(5) if (not self._steemd_process.returncode): self._rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '') else: raise Exception("steemd did not start properly...")
def validate(self): from steemapi.steemnoderpc import SteemNodeRPC if not Form.validate(self): return False try: SteemNodeRPC(self.node.data, user=self.rpcuser.data, password=self.rpcpass.data, num_retries=0) except: self.node.errors.append('Unable to connect') return False return True
def __init__(self, config): """ Initialize configuration """ available_features = dir(config) if ("wallet_host" in available_features and "wallet_port" in available_features): self.wallet_host = config.wallet_host self.wallet_port = config.wallet_port if ("wallet_user" in available_features and "wallet_password" in available_features): self.wallet_user = config.wallet_user self.wallet_password = config.wallet_password self.rpc = SteemWalletRPC(self.wallet_host, self.wallet_port, self.wallet_user, self.wallet_password) # Make a reference to 'wallet' self.wallet = self.rpc # Connect to Witness Node if "witness_url" in available_features: self.witness_url = config.witness_url if ("witness_user" in available_features): self.witness_user = config.witness_user if ("witness_password" in available_features): self.witness_password = config.witness_password self.ws = SteemNodeRPC(self.witness_url, self.witness_user, self.witness_password) # Make a reference to 'node' self.node = self.ws
def account_meta(account_name): rpc = SteemNodeRPC("wss://steemit.com/wspa") #try: #account_meta = json.loads(rpc.get_account(account_name)['json_metadata']) #account_url = account_meta['url'] last_tx, curr_tx = -1, -1 account_data, account_block, account_titles, account_links = [], [], [], [] while last_tx == curr_tx: curr_tx = curr_tx + 1 account_block = rpc.get_account_history(account_name, curr_tx, 0) last_tx = account_block[0][0] if account_block[0][1]['op'][0] == 'comment' and account_block[0][1][ 'op'][1]['author'] == account_name: #account_data.append(account_block) account_links.append(account_block[0][1]['op'][1]['permlink']) account_titles.append(account_block[0][1]['op'][1]['title']) print(str(account_titles[-1]) + ' - found your post!') accountset = set(account_titles) return (accountset)
def stream_blocks(server, block_nums, start, end): """Stream blocks from steemd in JSON format \b Which Steemd: \b 1. CLI "--server" option if provided 2. ENV var "WEBSOCKET_URL" if provided 3. Default: "wss://steemit.com/wspa" \b Which Blocks To Output: \b - Stream blocks beginning with current block by omitting --start, --end, and BLOCKS - Fetch a range of blocks using --start and/or --end - Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN) Where To Output Blocks: \b 2. ENV var "BLOCKS_OUT" if provided 3. Default: STDOUT """ # Setup steemd source rpc = SteemNodeRPC(server) with click.open_file('-', 'w', encoding='utf8') as f: if block_nums: blocks = _stream_blocks(rpc, block_nums) elif start and end: blocks = _stream_blocks(rpc, range(start, end)) else: blocks = rpc.block_stream(start) json_blocks = map(json.dumps, blocks) for block in json_blocks: click.echo(block, file=f)
def determine_user_index(account_name): #rpc = SteemNodeRPC("wss://steemit.com/wspa") try: try: [h_index, post_titles, sp_payouts ] = hindex_dict[account_name][str(datetime.date.today())] publish_date = str(datetime.date.today()) except: print('Could not find in dictionary: ' + account_name + ', on: ' + str(datetime.date.today())) rpc = SteemNodeRPC("ws://*****:*****@' + account_name + '/' + p for p in post_titles]) return render_template('index.html', name=account_name, body=body, post_list=post_list)
def __enter__( self ): self._steemd_lock.acquire() # Setup temp directory to use as the data directory for this self._temp_data_dir = TemporaryDirectory() config = Path( self._temp_data_dir.name ) / 'config.ini' config.touch() config.write_text( self._get_config() ) self._steemd_process = Popen( [ str( self._steemd_bin ), '--data-dir="' + str( self._temp_data_dir.name ) + '"' ], stdout=self._FNULL, stderr=self._FNULL ) self._steemd_process.poll() sleep( 5 ) if( not self._steemd_process.returncode ): self._rpc = SteemNodeRPC( 'ws://127.0.0.1:8090', '', '' ) else: raise Exception( "steemd did not start properly..." )
def determine_quick_user_index(account_name): rpc = SteemNodeRPC("wss://steemit.com/wspa") #rpc = SteemNodeRPC("ws://localhost:8090") post_payouts, post_votes, post_titles, post_links = author_rewards_quick( rpc, account_name) sp_payouts = rewards_conversion(post_payouts) h_index = 0 for pos, val in enumerate(sp_payouts): if val > pos + 1: h_index = pos + 1 else: break h2_index = 0 for pos, val in enumerate(sorted(post_votes, reverse=True)): if val > pos + 1: h2_index = pos + 1 else: break publish_date = str(datetime.date.today()) # return('V-index (reward value): ' + str(h_index) + '<br>P-index (popular votes): ' + str(h2_index) + '<hr>Most popular post: <a href=\"http://steemit.com' + post_links[0] + '\">' + post_titles[0] + '</a>, at a value of ' + str(sp_payouts[0]) + ' Steem Power.') body = 'V-index (reward value): ' + str( h_index) + '<br><br>P-index (popular votes): ' + str(h2_index) post_list = generate_table(sp_payouts, post_votes, post_titles, post_links) if post_list == "": return render_template('index.html', name=account_name, body=('<h1>Something went wrong. Has ' + account_name + ' ever posted?</h1>')) return render_template('index.html', name=account_name, body=body, post_list=post_list)
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs, max_threads): # pylint: disable=too-many-locals, too-many-statements rpc = SimpleSteemAPIClient(steemd_http_url) engine_config = configure_engine(database_url) db_name = engine_config.url.database db_user_name = engine_config.url.username Session.configure(bind=engine_config.engine) session = Session() # [1/7] confirm db connectivity task_message = fmt_task_message( 'Confirm database connectivity', emoji_code_point=u'\U0001F4DE', counter=1) click.echo(task_message) url, table_count = test_connection(database_url) if url: success_msg = fmt_success_message( 'connected to %s and found %s tables', url.__repr__(), table_count) click.echo(success_msg) if not url: raise Exception('Unable to connect to database') del url del table_count # [2/7] kill existing db threads task_message = fmt_task_message( 'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2) click.echo(task_message) all_procs, killed_procs = kill_db_processes(database_url, db_name, db_user_name) if len(killed_procs) > 0: success_msg = fmt_success_message('killed %s processes', len(killed_procs)) click.echo(success_msg) del all_procs del killed_procs # [3/7] init db if required task_message = fmt_task_message( 'Initialising db if required', emoji_code_point=u'\U0001F50C', counter=3) click.echo(task_message) init_tables(database_url, Base.metadata) # [4/7] find last irreversible block last_chain_block = rpc.last_irreversible_block_num() task_message = fmt_task_message( 'Finding highest blockchain block', emoji_code_point='\U0001F50E', counter=4) click.echo(task_message) success_msg = fmt_success_message( 'learned highest irreversible block is %s', last_chain_block) click.echo(success_msg) # [5/7] get missing block_nums task_message = fmt_task_message( 'Finding blocks missing from db', emoji_code_point=u'\U0001F52D', counter=5) click.echo(task_message) missing_block_nums_gen = Block.get_missing_block_num_iterator( session, last_chain_block, chunksize=100000) with click.progressbar( missing_block_nums_gen, label='Finding missing block_nums', color=True, show_eta=False, show_percent=False, empty_char='░', fill_char='█', show_pos=True, bar_template='%(bar)s %(info)s') as pbar: all_missing_block_nums = [] for missing_gen in pbar: all_missing_block_nums.extend(missing_gen()) success_msg = fmt_success_message('found %s missing blocks', len(all_missing_block_nums)) click.echo(success_msg) del missing_block_nums_gen del pbar session.invalidate() # [6/7] adding missing blocks task_message = fmt_task_message( 'Adding missing blocks to db, this may take a while', emoji_code_point=u'\U0001F4DD', counter=6) click.echo(task_message) max_workers = max_procs or os.cpu_count() or 1 chunksize = len(all_missing_block_nums) // max_workers if chunksize <= 0: chunksize = 1 map_func = partial( block_adder_process_worker, database_url, steemd_http_url, max_threads=max_threads) chunks = chunkify(all_missing_block_nums, 10000) with concurrent.futures.ProcessPoolExecutor( max_workers=max_workers) as executor: executor.map(map_func, chunks , chunksize=1) success_msg = fmt_success_message('added missing blocks') click.echo(success_msg) del all_missing_block_nums # [7/7] stream blocks task_message = fmt_task_message( 'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7) click.echo(task_message) highest_db_block = Block.highest_block(session) ws_rpc = SteemNodeRPC(steemd_websocket_url) blocks = ws_rpc.block_stream(highest_db_block) add_blocks(blocks, session)
class DebugNode( object ): """ Wraps the steemd debug node plugin for easier automated testing of the Steem Network""" def __init__( self, steemd, data_dir, args='', plugins=[], apis=[], steemd_out=None, steemd_err=None ): """ Creates a steemd debug node. It can be ran by using 'with debug_node:' While in the context of 'with' the debug node will continue to run. Upon exit of 'with' the debug will exit and clean up temporary files. This class also contains methods to allow basic manipulation of the blockchain. For all other requests, the python-steem library should be used. args: steemd -- The string path to the location of the steemd binary data_dir -- The string path to an existing steemd data directory which will be used to pull blocks from. args -- Other string args to pass to steemd. plugins -- Any additional plugins to start with the debug node. Modify plugins DebugNode.plugins apis -- Any additional APIs to have available. APIs will retain this order for accesibility starting at id 3. database_api is 0, login_api is 1, and debug_node_api is 2. Modify apis with DebugNode.api steemd_stdout -- A stream for steemd's stdout. Default is to pipe to /dev/null steemd_stderr -- A stream for steemd's stderr. Default is to pipe to /dev/null """ self._data_dir = None self._debug_key = None self._FNULL = None self._rpc = None self._steemd_bin = None self._steemd_lock = None self._steemd_process = None self._temp_data_dir = None self._steemd_bin = Path( steemd ) if( not self._steemd_bin.exists() ): raise ValueError( 'steemd does not exist' ) if( not self._steemd_bin.is_file() ): raise ValueError( 'steemd is not a file' ) self._data_dir = Path( data_dir ) if( not self._data_dir.exists() ): raise ValueError( 'data_dir either does not exist or is not a properly constructed steem data directory' ) if( not self._data_dir.is_dir() ): raise ValueError( 'data_dir is not a directory' ) self.plugins = plugins self.apis = apis if( args != '' ): self._args = args.split( "\\s" ) else: self._args = list() self._FNULL = open( devnull, 'w' ) if( steemd_out != None ): self.steemd_out = steemd_out else: self.steemd_out = self._FNULL if( steemd_err != None ): self.steemd_err = steemd_err else: self.steemd_err = self._FNULL self._debug_key = '5JHNbFNDg834SFj8CMArV6YW7td4zrPzXveqTfaShmYVuYNeK69' self._steemd_lock = Lock() def __enter__( self ): self._steemd_lock.acquire() # Setup temp directory to use as the data directory for this self._temp_data_dir = TemporaryDirectory() for child in self._data_dir.iterdir(): if( child.is_dir() ): copytree( str( child ), str( self._temp_data_dir.name ) + '/' + child.name ) db_version = Path( self._data_dir.name ) / 'db_version' if( db_version.exists() and not db_version.is_dir() ): copy2( str( db_version ), str( self._temp_data_dir.name ) + '/db_version' ) config = Path( self._temp_data_dir.name ) / 'config.ini' config.touch() config.write_text( self._get_config() ) steemd = [ str( self._steemd_bin ), '--data-dir=' + str( self._temp_data_dir.name ) ] steemd.extend( self._args ) self._steemd_process = Popen( steemd, stdout=self.steemd_out, stderr=self.steemd_err ) self._steemd_process.poll() sleep( 5 ) if( not self._steemd_process.returncode ): self._rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' ) else: raise Exception( "steemd did not start properly..." ) def __exit__( self, exc, value, tb ): self._rpc = None if( self._steemd_process != None ): self._steemd_process.poll() if( not self._steemd_process.returncode ): self._steemd_process.send_signal( SIGINT ) sleep( 7 ) self._steemd_process.poll() if( not self._steemd_process.returncode ): self._steemd_process.send_signal( SIGTERM ) sleep( 5 ) self._steemd_process.poll() if( self._steemd_process.returncode ): loggin.error( 'steemd did not properly shut down after SIGINT and SIGTERM. User intervention may be required.' ) self._steemd_process = None self._temp_data_dir.cleanup() self._temp_data_dir = None self._steemd_lock.release() def _get_config( self ): return "# no seed-node in config file or command line\n" \ + "p2p-endpoint = 127.0.0.1:2001 # bind to localhost to prevent remote p2p nodes from connecting to us\n" \ + "rpc-endpoint = 127.0.0.1:8095 # bind to localhost to secure RPC API access\n" \ + "enable-plugin = witness debug_node " + " ".join( self.plugins ) + "\n" \ + "public-api = database_api login_api debug_node_api " + " ".join( self.apis ) + "\n" def debug_generate_blocks( self, count ): """ Generate blocks on the current chain. Pending transactions will be applied, otherwise the blocks will be empty. The debug node plugin requires a WIF key to sign blocks with. This class uses the key 5JHNbFNDg834SFj8CMArV6YW7td4zrPzXveqTfaShmYVuYNeK69 which was generated from `get_dev_key steem debug`. Do not use this key on the live chain for any reason. args: count -- The number of new blocks to generate. returns: int: The number of blocks actually pushed. """ if( count < 0 ): raise ValueError( "count must be a positive non-zero number" ) return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_generate_blocks",["' + self._debug_key + '",' + str( count ) + ']], "id": 1}' ) ) def debug_generate_blocks_until( self, timestamp, generate_sparsely=True ): """ Generate block up until a head block time rather than a specific number of blocks. As with `debug_generate_blocks` all blocks will be empty unless there were pending transactions. The debug node plugin requires a WIF key to sign blocks with. This class uses the key 5JHNbFNDg834SFj8CMArV6YW7td4zrPzXveqTfaShmYVuYNeK69 which was generated from `get_dev_key steem debug`. Do not use this key on the live chain for any reason. args: time -- The desired new head block time. This is a POSIX Timestmap. generate_sparsely -- True if you wish to skip all intermediate blocks between the current head block time and the desired head block time. This is useful to trigger events, such as payouts and bandwidth updates, without generating blocks. However, many automatic chain updates (such as block inflation) will not continue at their normal rate as they are only calculated when a block is produced. returns: (time, int): A tuple including the new head block time and the number of blocks that were generated. """ if( not isinstance( timestamp, int ) ): raise ValueError( "Time must be a int" ) generate_sparsely_str = "true" if( not generate_sparsely ): generate_sparsely_str = "false" iso_string = datetime.fromtimestamp( timestamp, timezone.utc ).isoformat().split( '+' )[0].split( '-' ) if( len( iso_string ) == 4 ): iso_string = iso_string[:-1] iso_string = '-'.join( iso_string ) print( iso_string ) return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_generate_blocks_until",["' + self._debug_key + '","' + iso_string + '","' + generate_sparsely_str + '"]], "id": 1}' ) ) def debug_set_hardfork( self, hardfork_id ): """ Schedules a hardfork to happen on the next block. call `debug_generate_blocks( 1 )` to trigger the hardfork. All hardforks with id less than or equal to hardfork_id will be scheduled and triggered. args: hardfork_id: The id of the hardfork to set. Hardfork IDs start at 1 (0 is genesis) and increment by one for each hardfork. The maximum value is STEEM_NUM_HARDFORKS in chain/hardfork.d/0-preamble.hf """ if( hardfork_id < 0 ): raise ValueError( "hardfork_id cannot be negative" ) self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_set_hardfork",[' + str( hardfork_id ) + ']], "id":1}' ) ) def debug_has_hardfork( self, hardfork_id ): return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_has_hardfork",[' + str( hardfork_id ) + ']], "id":1}' ) ) def debug_get_witness_schedule( self ): return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_get_witness_schedule",[]], "id":1}' ) ) def debug_get_hardfork_property_object( self ): return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_get_hardfork_property_object",[]], "id":1}' ) )
class SteemClient() : """ The ``SteemClient`` class is an abstraction layer that makes the use of the RPC and the websocket interface easier to use. A part of this abstraction layer is to simplyfy the usage of objects and have an internal objects map updated to reduce unecessary queries (for enabled websocket connections). Advanced developers are of course free to use the underlying API classes instead as well. :param class config: the configuration class If a websocket connection is configured, the websocket subsystem can be run by: .. code-block:: python steem = SteemClient(config) steem.run() """ wallet_host = None wallet_port = None wallet_user = None wallet_password = None witness_url = None witness_user = None witness_password = None prefix = None #: RPC connection to the cli-wallet rpc = None wallet = None #: Websocket connection to the witness/full node ws = None node = None def __init__(self, config): """ Initialize configuration """ available_features = dir(config) if ("wallet_host" in available_features and "wallet_port" in available_features): self.wallet_host = config.wallet_host self.wallet_port = config.wallet_port if ("wallet_user" in available_features and "wallet_password" in available_features): self.wallet_user = config.wallet_user self.wallet_password = config.wallet_password self.rpc = SteemWalletRPC(self.wallet_host, self.wallet_port, self.wallet_user, self.wallet_password) # Make a reference to 'wallet' self.wallet = self.rpc # Connect to Witness Node if "witness_url" in available_features: self.witness_url = config.witness_url if ("witness_user" in available_features): self.witness_user = config.witness_user if ("witness_password" in available_features): self.witness_password = config.witness_password self.ws = SteemNodeRPC(self.witness_url, self.witness_user, self.witness_password) # Make a reference to 'node' self.node = self.ws def getObject(self, oid): """ Get an Object either from Websocket store (if available) or from RPC connection. """ if self.ws : return self.ws.get_object(oid)[0] else : return self.rpc.get_object(oid)[0]
# thanks to @furion @bitcalm @xeroc and @trogdor for inspiring the code from steemapi.steemnoderpc import SteemNodeRPC rpc = SteemNodeRPC('ws://node.steem.ws') from collections import Counter import csv # September 27 - beginning of the day - block 5320393 # October 3 - end of the day - block 5521856 voters = [ 'blocktrades', 'jamesc', 'smooth', 'dantheman', 'tombstone', 'summon', 'steemed', 'rainman', 'wang', 'complexring', 'riverhead', 'roadscape', 'nextgencrypto', 'silversteem', 'donkeypong', 'proskynneo', 'blackjack', 'firstclass', 'enki', 'clayop', 'wackou', 'steemit200', 'kushed', 'xeldal', 'arhag', 'fuzzyvest', 'pharesim', 'steempty', 'bitcube', 'onceuponatime', 'witnes.svk', 'abit', 'berniesanders', 'ned', 'dan', 'val-a', 'itsascam', 'skywalker', 'smooth.witness', 'datasecuritynode', 'au1nethyb1', 'bitcoin2016', 'analisa', 'paladin', 'satoshifund', 'alphabet', 'excalibur', 'recursive', 'lafona-miner', 'badassmother', 'xeroc' ] voterlst = [] authorlst = [] permlinklst = [] # parsing the blocks between Sept 27 - Oct 3 (end of day) for i in range(5320393, 5521856): dys = rpc.get_block(i)['transactions'] for tx in dys:
def __call__(self, form, field): from steemapi.steemnoderpc import SteemNodeRPC try: SteemNodeRPC(field.data, num_retries=0) except: raise ValidationError(self.message)
from datetime import datetime, timedelta from steemapi.steemnoderpc import SteemNodeRPC from piston.steem import Post from pymongo import MongoClient from pprint import pprint import json import time import sys import os rpc = SteemNodeRPC("ws://" + os.environ['steemnode'], "", "") mongo = MongoClient("mongodb://mongo") db = mongo.steemdb init = db.status.find_one({'_id': 'height'}) if (init): last_block = init['value'] else: last_block = 1 # ------------ # For development: # # If you're looking for a faster way to sync the data and get started, # uncomment this line with a more recent block, and the chain will start # to sync from that point onwards. Great for a development environment # where you want some data but don't want to sync the entire blockchain. # ------------ # last_block = 4155048
from datetime import datetime from steemapi.steemnoderpc import SteemNodeRPC from piston.steem import Post from pymongo import MongoClient from pprint import pprint import collections import time import sys import os from apscheduler.schedulers.background import BackgroundScheduler # rpc = SteemNodeRPC(host, "", "", ['follow_api']) rpc = SteemNodeRPC("ws://" + os.environ['steemnode'], "", "", apis=["follow", "database"]) mongo = MongoClient("mongodb://mongo") db = mongo.steemdb mvest_per_account = {} def load_accounts(): pprint("SteemDB - Loading mvest per account") for account in db.account.find(): if "name" in account.keys(): mvest_per_account.update( {account['name']: account['vesting_shares']})
from pprint import pprint import time """ Requires installation of xeroc's python-steemlib Adapted from monitor.py Connection Parameters to steemd daemon. Start the steemd daemon with the rpc-endpoint parameter: ./programs/steemd/steemd --rpc-endpoint=127.0.0.1:8092 This opens up a RPC port (e.g. at 8092). Currently, authentication is not yet available, thus, we recommend to restrict access to localhost. Later we will allow authentication via username and passpword (both empty now). """ rpc = SteemNodeRPC("ws://localhost:8090", "", "") """ Last Block that you have process in your backend. Processing will continue at `last_block + 1` """ last_block = 1160000 """ Witness account to monitor """ watch_account = "delegate.lafona" if __name__ == '__main__': # Let's find out how often blocks are generated!
from steemapi.steemnoderpc import SteemNodeRPC from pprint import pprint rpc = SteemNodeRPC("wss://steemit.com/ws") for a in rpc.stream("comment", start=1893850): pprint(a)
class DebugNode( object ): """ Wraps the steemd debug node plugin for easier automated testing of the Steem Network""" def __init__( self, steemd, data_dir, plugins=[], apis=[], steemd_stdout=None, steemd_stderr=None ): """ Creates a steemd debug node. It can be ran by using 'with debug_node:' While in the context of 'with' the debug node will continue to run. Upon exit of 'with' the debug will exit and clean up temporary files. This class also contains methods to allow basic manipulation of the blockchain. For all other requests, the python-steem library should be used. args: steemd -- The string path to the location of the steemd binary data_dir -- The string path to an existing steemd data directory which will be used to pull blocks from. plugins -- Any additional plugins to start with the debug node. Modify plugins DebugNode.plugins apis -- Any additional APIs to have available. APIs will retain this order for accesibility starting at id 3. database_api is 0, login_api is 1, and debug_node_api is 2. Modify apis with DebugNode.api steemd_stdout -- A stream for steemd's stdout. Default is to pipe to /dev/null steemd_stderr -- A stream for steemd's stderr. Default is to pipe to /dev/null """ self._block_dir = None self._debug_key = None self._FNULL = None self._rpc = None self._steemd_bin = None self._steemd_lock = None self._steemd_process = None self._temp_data_dir = None self._steemd_bin = Path( steemd ) if( not self._steemd_bin.exists() ): raise ValueError( 'steemd does not exist' ) if( not self._steemd_bin.is_file() ): raise ValueError( 'steemd is not a file' ) self._block_dir = Path( data_dir ) / 'blockchain/database/block_num_to_block' if( not self._block_dir.exists() ): raise ValueError( 'data_dir either does not exist or is not a properly constructed steem data directory' ) if( not self._block_dir.is_dir() ): raise ValueError( 'data_dir is not a directory' ) self.plugins = plugins self.apis = apis self._FNULL = open( devnull, 'w' ) if( steemd_stdout != None ): self.steemd_stdout = steemd_stdout else: self.steemd_stdout = self._FNULL if( steemd_stderr != None ): self.steemd_stderr = steemd_stderr else: self.steemd_stderr = self._FNULL self._debug_key = '5JHNbFNDg834SFj8CMArV6YW7td4zrPzXveqTfaShmYVuYNeK69' self._steemd_lock = Lock() def __enter__( self ): self._steemd_lock.acquire() # Setup temp directory to use as the data directory for this self._temp_data_dir = TemporaryDirectory() config = Path( self._temp_data_dir.name ) / 'config.ini' config.touch() config.write_text( self._get_config() ) self._steemd_process = Popen( [ str( self._steemd_bin ), '--data-dir="' + str( self._temp_data_dir.name ) + '"' ], stdout=self._FNULL, stderr=self._FNULL ) self._steemd_process.poll() sleep( 5 ) if( not self._steemd_process.returncode ): self._rpc = SteemNodeRPC( 'ws://127.0.0.1:8090', '', '' ) else: raise Exception( "steemd did not start properly..." ) def __exit__( self, exc, value, tb ): self._rpc = None if( self._steemd_process != None ): self._steemd_process.poll() if( not self._steemd_process.returncode ): self._steemd_process.send_signal( SIGINT ) sleep( 7 ) self._steemd_process.poll() if( not self._steemd_process.returncode ): self._steemd_process.send_signal( SIGTERM ) sleep( 5 ) self._steemd_process.poll() if( self._steemd_process.returncode ): loggin.error( 'steemd did not properly shut down after SIGINT and SIGTERM. User intervention may be required.' ) self._steemd_process = None self._temp_data_dir.cleanup() self._temp_data_dir = None self._steemd_lock.release() def _get_config( self ): return "# no seed-node in config file or command line\n" \ + "p2p-endpoint = 127.0.0.1:2001 # bind to localhost to prevent remote p2p nodes from connecting to us\n" \ + "rpc-endpoint = 127.0.0.1:8090 # bind to localhost to secure RPC API access\n" \ + "enable-plugin = witness debug_node " + " ".join( self.plugins ) + "\n" \ + "public-api = database_api login_api debug_node_api " + " ".join( self.apis ) + "\n" def debug_push_blocks( self, count=0 ): """ Push count blocks from an existing chain. There is no guarantee pushing blocks will work depending on set hardforks, or generated blocks that may change chain state significantly from what is is in the original data directory. It is recommend to not call `debug_push_blocks` after making any changes to the chain state. args: count -- The number of blocks to push. Default is 0 which will push all blocks. returns: int: The number of blocks actually pushed. """ num_blocks = 0 if( count == 0 ): ret = 10000 while( ret == 10000 ): ret = self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_push_blocks",["' + str( self._block_dir ) + '", 10000]], "id": 1}' ) ) num_blocks += ret else: ret = self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_push_blocks",["' + str( self._block_dir ) + '",' + str( count ) + ']], "id": 1}' ) ) num_blocks += ret return num_blocks def debug_generate_blocks( self, count ): """ Generate blocks on the current chain. Pending transactions will be applied, otherwise the blocks will be empty. The debug node plugin requires a WIF key to sign blocks with. This class uses the key 5JHNbFNDg834SFj8CMArV6YW7td4zrPzXveqTfaShmYVuYNeK69 which was generated from `get_dev_key steem debug`. Do not use this key on the live chain for any reason. args: count -- The number of new blocks to generate. returns: int: The number of blocks actually pushed. """ if( count < 0 ): raise ValueError( "count must be a positive non-zero number" ) return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_generate_blocks",["' + self._debug_key + '",' + str( count ) + ']], "id": 1}' ) ) def debug_generate_blocks_until( self, time, generate_sparsely=True ): """ Generate block up until a head block time rather than a specific number of blocks. As with `debug_generate_blocks` all blocks will be empty unless there were pending transactions. The debug node plugin requires a WIF key to sign blocks with. This class uses the key 5JHNbFNDg834SFj8CMArV6YW7td4zrPzXveqTfaShmYVuYNeK69 which was generated from `get_dev_key steem debug`. Do not use this key on the live chain for any reason. args: time -- The desired new head block time. This is a UTC Timestmap. generate_sparsely -- True if you wish to skip all intermediate blocks between the current head block time and the desired head block time. This is useful to trigger events, such as payouts and bandwidth updates, without generating blocks. However, many automatic chain updates (such as block inflation) will not continue at their normal rate as they are only calculated when a block is produced. returns: (time, int): A tuple including the new head block time and the number of blocks that were generated. """ if( not instance( time, datetime ) ): raise ValueError( "Time must be a datetime object" ) self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_generate_blocks_until",["' + self._debug_key + '",' + str( time ) + ']], "id": 1}' ) ) def debug_set_hardfork( self, hardfork_id ): """ Schedules a hardfork to happen on the next block. call `debug_generate_blocks( 1 )` to trigger the hardfork. All hardforks with id less than or equal to hardfork_id will be scheduled and triggered. args: hardfork_id: The id of the hardfork to set. Hardfork IDs start at 1 (0 is genesis) and increment by one for each hardfork. The maximum value is STEEMIT_NUM_HARDFORKS in chain/hardfork.d/0-preamble.hf """ if( hardfork_id < 0 ): raise ValueError( "hardfork_id cannot be negative" ) self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_set_hardfork",[' + str( hardfork_id ) + ']], "id":1}' ) )
import time import os """ Requires installation of xeroc's python-steemlib Adapted from monitor.py Connection Parameters to steemd daemon. Start the steemd daemon with the rpc-endpoint parameter: ./programs/steemd/steemd --rpc-endpoint=127.0.0.1:8092 This opens up a RPC port (e.g. at 8092). Currently, authentication is not yet available, thus, we recommend to restrict access to localhost. Later we will allow authentication via username and passpword (both empty now). """ rpc = SteemNodeRPC("ws://localhost:8090", "", "") """ Last Block that you have process in your backend. Processing will continue at `last_block + 1` """ last_block = 1 """ Witness accounts to monitor """ watch_accounts = ["roadscape", "kushed", "smooth.witness", "abit", "pharesim", "dele-puppy", "steemed", "nextgencrypto", "clayop", "au1nethyb1", "steempty", "riverhead", "arhag", "complexring", "bitcube", "silversteem", "xeldal", "witness.svk", "wackou", "bhuz", "steemychicken1", "datasecuritynode", "cyrano.witness", "delegate.lafona", "boatymcboatface", "jabbasteem", "bue", "ihashfury", "masteryoda", "joseph", "pumpkin", "blocktrades", "steem-id", "liondani", "salvation", "modprobe", "dantheman", "mrs.agsexplorer", "fminerten", "lafona", "pfunk", "summon", "randaletouri", "rainman", "itsascam", "steemroller", "hello", "lxcteem", "steemit", "idol", "bittrexrichie", "testzcrypto", "moon", "drsteem", "afew", "phantas", "healthcare", "felekas1", "abka", "neonminer", "silkroad", "signalandnoise", "stephanie", "stesting1", "drifter1", "support1", "alittle", "zisis1", "miltos1"] if __name__ == '__main__': # Let's find out how often blocks are generated!
from pprint import pprint import time """ Connection Parameters to steemd daemon. Start the steemd daemon with the rpc-endpoint parameter: ./programs/steemd/steemd --rpc-endpoint=127.0.0.1:8092 This opens up a RPC port (e.g. at 8092). Currently, authentication is not yet available, thus, we recommend to restrict access to localhost. Later we will allow authentication via username and passpword (both empty now). """ rpc = SteemNodeRPC("ws://localhost:8090", "", "") """ Last Block that you have process in your backend. Processing will continue at `last_block + 1` """ last_block = 160900 """ Deposit account name to monitor """ watch_account = "world" def process_block(block, blockid): """ This call processes a block which can carry many transactions
def __init__(self): self.rpc = SteemNodeRPC("wss://node.steem.ws", "", "", apis=["follow"])
# thanks @furion for the tip from steemapi.steemnoderpc import SteemNodeRPC rpc = SteemNodeRPC('ws://node.steem.ws') import csv allauthors = rpc.lookup_accounts('', 1000000) postdict = dict() repdict = dict() rewdict = dict() # counting author posts and adding them to the postdict dictionary # retrieving authors with more than 700 posts (blogposts and comments) for author in allauthors: postcount = rpc.get_account(author)['post_count'] if postcount > 700: postdict[author] = postcount # retrieving reputation and post rewards for the authors in postdict for author2 in postdict.keys(): repcount = rpc.get_account(author2)['reputation'] rewcount = rpc.get_account(author2)['posting_rewards'] repdict[author2] = repcount rewdict[author2] = rewcount # indexing results for posts writefile1 = open('productivesteemians-posts.csv', 'w', newline='') writer1 = csv.writer(writefile1)
from datetime import datetime, timedelta from steemapi.steemnoderpc import SteemNodeRPC from piston.steem import Post from pymongo import MongoClient from pprint import pprint import collections import json import time import sys import os rpc = SteemNodeRPC("ws://" + os.environ['steemnode'], "", "", apis=["follow", "database"]) mongo = MongoClient("mongodb://mongo") db = mongo.steemdb init = db.status.find_one({'_id': 'height'}) if (init): last_block = init['value'] else: last_block = 1 # ------------ # For development: # # If you're looking for a faster way to sync the data and get started, # uncomment this line with a more recent block, and the chain will start # to sync from that point onwards. Great for a development environment # where you want some data but don't want to sync the entire blockchain.