def __enter__( self ): self._steemd_lock.acquire() # Setup temp directory to use as the data directory for this self._temp_data_dir = TemporaryDirectory() for child in self._data_dir.iterdir(): if( child.is_dir() ): copytree( str( child ), str( self._temp_data_dir.name ) + '/' + child.name ) db_version = Path( self._data_dir.name ) / 'db_version' if( db_version.exists() and not db_version.is_dir() ): copy2( str( db_version ), str( self._temp_data_dir.name ) + '/db_version' ) config = Path( self._temp_data_dir.name ) / 'config.ini' config.touch() config.write_text( self._get_config() ) steemd = [ str( self._steemd_bin ), '--data-dir=' + str( self._temp_data_dir.name ) ] steemd.extend( self._args ) self._steemd_process = Popen( steemd, stdout=self.steemd_out, stderr=self.steemd_err ) self._steemd_process.poll() sleep( 5 ) if( not self._steemd_process.returncode ): self._rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' ) else: raise Exception( "steemd did not start properly..." )
def run_steemd_tests( debug_node ): from steemapi.steemnoderpc import SteemNodeRPC try: print( 'Replaying blocks...', ) sys.stdout.flush() total_blocks = 0 while( total_blocks % 100000 == 0 ): total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True ) print( 'Blocks Replayed: ' + str( total_blocks ) ) sys.stdout.flush() print( "Triggering payouts" ) sys.stdout.flush() debug_node.debug_generate_blocks_until( 1467590400 - 3 ) rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' ) ret = rpc.lookup_accounts( '', str( 0xFFFFFFFF ) ) account_balances = {} for acc_name in ret: acc = rpc.get_accounts( [ acc_name ] ) steemd = float( acc[0][ 'balance' ].split( ' ' )[0] ) sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] ) vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] ) account_balances[ acc_name ] = ( steemd, sbd, vests ) debug_node.debug_generate_blocks( 1 ) account_rewards = {} for acc_name, bal in account_balances.items(): acc = rpc.get_accounts( [ acc_name ] ) steemd = float( acc[0][ 'balance' ].split( ' ' )[0] ) - bal[0] sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] ) - bal[1] vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] ) - bal[2] account_rewards[ acc_name ] = ( steemd, sbd, vests ) print( "Generating blocks to verify nothing broke" ) assert( debug_node.debug_generate_blocks( 10 ) == 10 ) ''' print( "Done!" ) print( "Getting comment dump:" ) sys.stdout.flush() ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) ); print( 'author, url, total_payout_value, abs_rshares, num_active_votes' ) for comment in ret: print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] ) ''' print( "Printing account reward dump:" ) print( "account, steem, sbd, vests" ) for acc_name, rew in account_rewards.items(): print( acc_name + ', ' + str( rew[0] ) + ' STEEM, ' + str( rew[1] ) + ' SBD, ' + str( rew[2] ) + ' VESTS' ) except ValueError as val_err: print( str( val_err ) )
def run_steemd_tests(debug_node): from steemapi.steemnoderpc import SteemNodeRPC try: print('Replaying blocks...', ) sys.stdout.flush() total_blocks = 0 while (total_blocks % 100000 == 0): total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True) print('Blocks Replayed: ' + str(total_blocks)) sys.stdout.flush() print("Triggering payouts") sys.stdout.flush() debug_node.debug_generate_blocks_until(1467590400 - 3) rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '') ret = rpc.lookup_accounts('', str(0xFFFFFFFF)) debug_node.debug_generate_blocks(1) print("Generating blocks to verify nothing broke") assert (debug_node.debug_generate_blocks(10) == 10) account_rewards = {} vote_count = {} for acc_name in ret: acc = rpc.get_accounts([acc_name]) #print( acc_name + ',' + acc[0][ 'curation_rewards' ] ) account_rewards[acc_name] = float( acc[0]['curation_rewards'].split(' ')[0]) vote_count[acc_name] = int(acc[0]['lifetime_vote_count']) ''' print( "Done!" ) print( "Getting comment dump:" ) sys.stdout.flush() ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) ); print( 'author, url, total_payout_value, abs_rshares, num_active_votes' ) for comment in ret: print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] ) ''' print("Printing account reward dump:") sorted_rewards = sorted(account_rewards.items(), key=operator.itemgetter(1)) print("account, curation_steem") for rew in sorted_rewards: print(rew[0] + ', ' + str(rew[1]) + ', ' + str(vote_count[rew[0]])) except ValueError as val_err: print(str(val_err))
def validate(self): from steemapi.steemnoderpc import SteemNodeRPC if not Form.validate(self): return False try: SteemNodeRPC(self.node.data, user=self.rpcuser.data, password=self.rpcpass.data, num_retries=0) except: self.node.errors.append('Unable to connect') return False return True
def determine_user_index(account_name): #rpc = SteemNodeRPC("wss://steemit.com/wspa") try: try: [h_index, post_titles, sp_payouts ] = hindex_dict[account_name][str(datetime.date.today())] publish_date = str(datetime.date.today()) except: print('Could not find in dictionary: ' + account_name + ', on: ' + str(datetime.date.today())) rpc = SteemNodeRPC("ws://*****:*****@' + account_name + '/' + p for p in post_titles]) return render_template('index.html', name=account_name, body=body, post_list=post_list)
def determine_quick_user_index(account_name): rpc = SteemNodeRPC("wss://steemit.com/wspa") #rpc = SteemNodeRPC("ws://localhost:8090") post_payouts, post_votes, post_titles, post_links = author_rewards_quick( rpc, account_name) sp_payouts = rewards_conversion(post_payouts) h_index = 0 for pos, val in enumerate(sp_payouts): if val > pos + 1: h_index = pos + 1 else: break h2_index = 0 for pos, val in enumerate(sorted(post_votes, reverse=True)): if val > pos + 1: h2_index = pos + 1 else: break publish_date = str(datetime.date.today()) # return('V-index (reward value): ' + str(h_index) + '<br>P-index (popular votes): ' + str(h2_index) + '<hr>Most popular post: <a href=\"http://steemit.com' + post_links[0] + '\">' + post_titles[0] + '</a>, at a value of ' + str(sp_payouts[0]) + ' Steem Power.') body = 'V-index (reward value): ' + str( h_index) + '<br><br>P-index (popular votes): ' + str(h2_index) post_list = generate_table(sp_payouts, post_votes, post_titles, post_links) if post_list == "": return render_template('index.html', name=account_name, body=('<h1>Something went wrong. Has ' + account_name + ' ever posted?</h1>')) return render_template('index.html', name=account_name, body=body, post_list=post_list)
def __enter__(self): self._steemd_lock.acquire() # Setup temp directory to use as the data directory for this self._temp_data_dir = TemporaryDirectory() config = Path(self._temp_data_dir.name) / 'config.ini' config.touch() config.write_text(self._get_config()) self._steemd_process = Popen([ str(self._steemd_bin), '--data-dir="' + str(self._temp_data_dir.name) + '"' ], stdout=self.steemd_out, stderr=self.steemd_err) self._steemd_process.poll() sleep(5) if (not self._steemd_process.returncode): self._rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '') else: raise Exception("steemd did not start properly...")
def __init__(self, config): """ Initialize configuration """ available_features = dir(config) if ("wallet_host" in available_features and "wallet_port" in available_features): self.wallet_host = config.wallet_host self.wallet_port = config.wallet_port if ("wallet_user" in available_features and "wallet_password" in available_features): self.wallet_user = config.wallet_user self.wallet_password = config.wallet_password self.rpc = SteemWalletRPC(self.wallet_host, self.wallet_port, self.wallet_user, self.wallet_password) # Make a reference to 'wallet' self.wallet = self.rpc # Connect to Witness Node if "witness_url" in available_features: self.witness_url = config.witness_url if ("witness_user" in available_features): self.witness_user = config.witness_user if ("witness_password" in available_features): self.witness_password = config.witness_password self.ws = SteemNodeRPC(self.witness_url, self.witness_user, self.witness_password) # Make a reference to 'node' self.node = self.ws
def account_meta(account_name): rpc = SteemNodeRPC("wss://steemit.com/wspa") #try: #account_meta = json.loads(rpc.get_account(account_name)['json_metadata']) #account_url = account_meta['url'] last_tx, curr_tx = -1, -1 account_data, account_block, account_titles, account_links = [], [], [], [] while last_tx == curr_tx: curr_tx = curr_tx + 1 account_block = rpc.get_account_history(account_name, curr_tx, 0) last_tx = account_block[0][0] if account_block[0][1]['op'][0] == 'comment' and account_block[0][1][ 'op'][1]['author'] == account_name: #account_data.append(account_block) account_links.append(account_block[0][1]['op'][1]['permlink']) account_titles.append(account_block[0][1]['op'][1]['title']) print(str(account_titles[-1]) + ' - found your post!') accountset = set(account_titles) return (accountset)
def stream_blocks(server, block_nums, start, end): """Stream blocks from steemd in JSON format \b Which Steemd: \b 1. CLI "--server" option if provided 2. ENV var "WEBSOCKET_URL" if provided 3. Default: "wss://steemit.com/wspa" \b Which Blocks To Output: \b - Stream blocks beginning with current block by omitting --start, --end, and BLOCKS - Fetch a range of blocks using --start and/or --end - Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN) Where To Output Blocks: \b 2. ENV var "BLOCKS_OUT" if provided 3. Default: STDOUT """ # Setup steemd source rpc = SteemNodeRPC(server) with click.open_file('-', 'w', encoding='utf8') as f: if block_nums: blocks = _stream_blocks(rpc, block_nums) elif start and end: blocks = _stream_blocks(rpc, range(start, end)) else: blocks = rpc.block_stream(start) json_blocks = map(json.dumps, blocks) for block in json_blocks: click.echo(block, file=f)
from pprint import pprint import time """ Connection Parameters to steemd daemon. Start the steemd daemon with the rpc-endpoint parameter: ./programs/steemd/steemd --rpc-endpoint=127.0.0.1:8092 This opens up a RPC port (e.g. at 8092). Currently, authentication is not yet available, thus, we recommend to restrict access to localhost. Later we will allow authentication via username and passpword (both empty now). """ rpc = SteemNodeRPC("ws://localhost:8090", "", "") """ Last Block that you have process in your backend. Processing will continue at `last_block + 1` """ last_block = 160900 """ Deposit account name to monitor """ watch_account = "world" def process_block(block, blockid): """ This call processes a block which can carry many transactions
from datetime import datetime, timedelta from steemapi.steemnoderpc import SteemNodeRPC from piston.steem import Post from pymongo import MongoClient from pprint import pprint import json import time import sys import os rpc = SteemNodeRPC("ws://" + os.environ['steemnode'], "", "") mongo = MongoClient("mongodb://mongo") db = mongo.steemdb init = db.status.find_one({'_id': 'height'}) if (init): last_block = init['value'] else: last_block = 1 # ------------ # For development: # # If you're looking for a faster way to sync the data and get started, # uncomment this line with a more recent block, and the chain will start # to sync from that point onwards. Great for a development environment # where you want some data but don't want to sync the entire blockchain. # ------------ # last_block = 4155048
from datetime import datetime from steemapi.steemnoderpc import SteemNodeRPC from piston.steem import Post from pymongo import MongoClient from pprint import pprint import collections import time import sys import os from apscheduler.schedulers.background import BackgroundScheduler # rpc = SteemNodeRPC(host, "", "", ['follow_api']) rpc = SteemNodeRPC("ws://" + os.environ['steemnode'], "", "", apis=["follow", "database"]) mongo = MongoClient("mongodb://mongo") db = mongo.steemdb mvest_per_account = {} def load_accounts(): pprint("SteemDB - Loading mvest per account") for account in db.account.find(): if "name" in account.keys(): mvest_per_account.update( {account['name']: account['vesting_shares']})
# thanks @furion for the tip from steemapi.steemnoderpc import SteemNodeRPC rpc = SteemNodeRPC('ws://node.steem.ws') import csv allauthors = rpc.lookup_accounts('', 1000000) postdict = dict() repdict = dict() rewdict = dict() # counting author posts and adding them to the postdict dictionary # retrieving authors with more than 700 posts (blogposts and comments) for author in allauthors: postcount = rpc.get_account(author)['post_count'] if postcount > 700: postdict[author] = postcount # retrieving reputation and post rewards for the authors in postdict for author2 in postdict.keys(): repcount = rpc.get_account(author2)['reputation'] rewcount = rpc.get_account(author2)['posting_rewards'] repdict[author2] = repcount rewdict[author2] = rewcount # indexing results for posts writefile1 = open('productivesteemians-posts.csv', 'w', newline='') writer1 = csv.writer(writefile1)
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs, max_threads): # pylint: disable=too-many-locals, too-many-statements rpc = SimpleSteemAPIClient(steemd_http_url) engine_config = configure_engine(database_url) db_name = engine_config.url.database db_user_name = engine_config.url.username Session.configure(bind=engine_config.engine) session = Session() # [1/7] confirm db connectivity task_message = fmt_task_message( 'Confirm database connectivity', emoji_code_point=u'\U0001F4DE', counter=1) click.echo(task_message) url, table_count = test_connection(database_url) if url: success_msg = fmt_success_message( 'connected to %s and found %s tables', url.__repr__(), table_count) click.echo(success_msg) if not url: raise Exception('Unable to connect to database') del url del table_count # [2/7] kill existing db threads task_message = fmt_task_message( 'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2) click.echo(task_message) all_procs, killed_procs = kill_db_processes(database_url, db_name, db_user_name) if len(killed_procs) > 0: success_msg = fmt_success_message('killed %s processes', len(killed_procs)) click.echo(success_msg) del all_procs del killed_procs # [3/7] init db if required task_message = fmt_task_message( 'Initialising db if required', emoji_code_point=u'\U0001F50C', counter=3) click.echo(task_message) init_tables(database_url, Base.metadata) # [4/7] find last irreversible block last_chain_block = rpc.last_irreversible_block_num() task_message = fmt_task_message( 'Finding highest blockchain block', emoji_code_point='\U0001F50E', counter=4) click.echo(task_message) success_msg = fmt_success_message( 'learned highest irreversible block is %s', last_chain_block) click.echo(success_msg) # [5/7] get missing block_nums task_message = fmt_task_message( 'Finding blocks missing from db', emoji_code_point=u'\U0001F52D', counter=5) click.echo(task_message) missing_block_nums_gen = Block.get_missing_block_num_iterator( session, last_chain_block, chunksize=100000) with click.progressbar( missing_block_nums_gen, label='Finding missing block_nums', color=True, show_eta=False, show_percent=False, empty_char='░', fill_char='█', show_pos=True, bar_template='%(bar)s %(info)s') as pbar: all_missing_block_nums = [] for missing_gen in pbar: all_missing_block_nums.extend(missing_gen()) success_msg = fmt_success_message('found %s missing blocks', len(all_missing_block_nums)) click.echo(success_msg) del missing_block_nums_gen del pbar session.invalidate() # [6/7] adding missing blocks task_message = fmt_task_message( 'Adding missing blocks to db, this may take a while', emoji_code_point=u'\U0001F4DD', counter=6) click.echo(task_message) max_workers = max_procs or os.cpu_count() or 1 chunksize = len(all_missing_block_nums) // max_workers if chunksize <= 0: chunksize = 1 map_func = partial( block_adder_process_worker, database_url, steemd_http_url, max_threads=max_threads) chunks = chunkify(all_missing_block_nums, 10000) with concurrent.futures.ProcessPoolExecutor( max_workers=max_workers) as executor: executor.map(map_func, chunks , chunksize=1) success_msg = fmt_success_message('added missing blocks') click.echo(success_msg) del all_missing_block_nums # [7/7] stream blocks task_message = fmt_task_message( 'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7) click.echo(task_message) highest_db_block = Block.highest_block(session) ws_rpc = SteemNodeRPC(steemd_websocket_url) blocks = ws_rpc.block_stream(highest_db_block) add_blocks(blocks, session)
def __call__(self, form, field): from steemapi.steemnoderpc import SteemNodeRPC try: SteemNodeRPC(field.data, num_retries=0) except: raise ValidationError(self.message)
def __init__(self): self.rpc = SteemNodeRPC("wss://node.steem.ws", "", "", apis=["follow"])
from steemapi.steemnoderpc import SteemNodeRPC from pprint import pprint rpc = SteemNodeRPC("wss://steemit.com/ws") for a in rpc.stream("comment", start=1893850): pprint(a)