def __enter__( self ):
self._steemd_lock.acquire()
# Setup temp directory to use as the data directory for this
self._temp_data_dir = TemporaryDirectory()
for child in self._data_dir.iterdir():
if( child.is_dir() ):
copytree( str( child ), str( self._temp_data_dir.name ) + '/' + child.name )
db_version = Path( self._data_dir.name ) / 'db_version'
if( db_version.exists() and not db_version.is_dir() ):
copy2( str( db_version ), str( self._temp_data_dir.name ) + '/db_version' )
config = Path( self._temp_data_dir.name ) / 'config.ini'
config.touch()
config.write_text( self._get_config() )
steemd = [ str( self._steemd_bin ), '--data-dir=' + str( self._temp_data_dir.name ) ]
steemd.extend( self._args )
self._steemd_process = Popen( steemd, stdout=self.steemd_out, stderr=self.steemd_err )
self._steemd_process.poll()
sleep( 5 )
if( not self._steemd_process.returncode ):
self._rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' )
else:
raise Exception( "steemd did not start properly..." )
def run_steemd_tests( debug_node ):
from steemapi.steemnoderpc import SteemNodeRPC
try:
print( 'Replaying blocks...', )
sys.stdout.flush()
total_blocks = 0
while( total_blocks % 100000 == 0 ):
total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True )
print( 'Blocks Replayed: ' + str( total_blocks ) )
sys.stdout.flush()
print( "Triggering payouts" )
sys.stdout.flush()
debug_node.debug_generate_blocks_until( 1467590400 - 3 )
rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' )
ret = rpc.lookup_accounts( '', str( 0xFFFFFFFF ) )
account_balances = {}
for acc_name in ret:
acc = rpc.get_accounts( [ acc_name ] )
steemd = float( acc[0][ 'balance' ].split( ' ' )[0] )
sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] )
vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] )
account_balances[ acc_name ] = ( steemd, sbd, vests )
debug_node.debug_generate_blocks( 1 )
account_rewards = {}
for acc_name, bal in account_balances.items():
acc = rpc.get_accounts( [ acc_name ] )
steemd = float( acc[0][ 'balance' ].split( ' ' )[0] ) - bal[0]
sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] ) - bal[1]
vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] ) - bal[2]
account_rewards[ acc_name ] = ( steemd, sbd, vests )
print( "Generating blocks to verify nothing broke" )
assert( debug_node.debug_generate_blocks( 10 ) == 10 )
'''
print( "Done!" )
print( "Getting comment dump:" )
sys.stdout.flush()
ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) );
print( 'author, url, total_payout_value, abs_rshares, num_active_votes' )
for comment in ret:
print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] )
'''
print( "Printing account reward dump:" )
print( "account, steem, sbd, vests" )
for acc_name, rew in account_rewards.items():
print( acc_name + ', ' + str( rew[0] ) + ' STEEM, ' + str( rew[1] ) + ' SBD, ' + str( rew[2] ) + ' VESTS' )
except ValueError as val_err:
print( str( val_err ) )
def run_steemd_tests(debug_node):
from steemapi.steemnoderpc import SteemNodeRPC
try:
print('Replaying blocks...', )
sys.stdout.flush()
total_blocks = 0
while (total_blocks % 100000 == 0):
total_blocks += debug_node.debug_push_blocks(
100000, skip_validate_invariants=True)
print('Blocks Replayed: ' + str(total_blocks))
sys.stdout.flush()
print("Triggering payouts")
sys.stdout.flush()
debug_node.debug_generate_blocks_until(1467590400 - 3)
rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '')
ret = rpc.lookup_accounts('', str(0xFFFFFFFF))
debug_node.debug_generate_blocks(1)
print("Generating blocks to verify nothing broke")
assert (debug_node.debug_generate_blocks(10) == 10)
account_rewards = {}
vote_count = {}
for acc_name in ret:
acc = rpc.get_accounts([acc_name])
#print( acc_name + ',' + acc[0][ 'curation_rewards' ] )
account_rewards[acc_name] = float(
acc[0]['curation_rewards'].split(' ')[0])
vote_count[acc_name] = int(acc[0]['lifetime_vote_count'])
'''
print( "Done!" )
print( "Getting comment dump:" )
sys.stdout.flush()
ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) );
print( 'author, url, total_payout_value, abs_rshares, num_active_votes' )
for comment in ret:
print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] )
'''
print("Printing account reward dump:")
sorted_rewards = sorted(account_rewards.items(),
key=operator.itemgetter(1))
print("account, curation_steem")
for rew in sorted_rewards:
print(rew[0] + ', ' + str(rew[1]) + ', ' + str(vote_count[rew[0]]))
except ValueError as val_err:
print(str(val_err))
def validate(self):
from steemapi.steemnoderpc import SteemNodeRPC
if not Form.validate(self):
return False
try:
SteemNodeRPC(self.node.data,
user=self.rpcuser.data,
password=self.rpcpass.data,
num_retries=0)
except:
self.node.errors.append('Unable to connect')
return False
return True
def determine_user_index(account_name):
#rpc = SteemNodeRPC("wss://steemit.com/wspa")
try:
try:
[h_index, post_titles, sp_payouts
] = hindex_dict[account_name][str(datetime.date.today())]
publish_date = str(datetime.date.today())
except:
print('Could not find in dictionary: ' + account_name + ', on: ' +
str(datetime.date.today()))
rpc = SteemNodeRPC("ws://*****:*****@' + account_name + '/' + p for p in post_titles])
return render_template('index.html',
name=account_name,
body=body,
post_list=post_list)
def determine_quick_user_index(account_name):
rpc = SteemNodeRPC("wss://steemit.com/wspa")
#rpc = SteemNodeRPC("ws://localhost:8090")
post_payouts, post_votes, post_titles, post_links = author_rewards_quick(
rpc, account_name)
sp_payouts = rewards_conversion(post_payouts)
h_index = 0
for pos, val in enumerate(sp_payouts):
if val > pos + 1:
h_index = pos + 1
else:
break
h2_index = 0
for pos, val in enumerate(sorted(post_votes, reverse=True)):
if val > pos + 1:
h2_index = pos + 1
else:
break
publish_date = str(datetime.date.today())
# return('V-index (reward value): ' + str(h_index) + '<br>P-index (popular votes): ' + str(h2_index) + '<hr>Most popular post: <a href=\"http://steemit.com' + post_links[0] + '\">' + post_titles[0] + '</a>, at a value of ' + str(sp_payouts[0]) + ' Steem Power.')
body = 'V-index (reward value): ' + str(
h_index) + '<br><br>P-index (popular votes): ' + str(h2_index)
post_list = generate_table(sp_payouts, post_votes, post_titles, post_links)
if post_list == "":
return render_template('index.html',
name=account_name,
body=('<h1>Something went wrong. Has ' +
account_name + ' ever posted?</h1>'))
return render_template('index.html',
name=account_name,
body=body,
post_list=post_list)
def __enter__(self):
self._steemd_lock.acquire()
# Setup temp directory to use as the data directory for this
self._temp_data_dir = TemporaryDirectory()
config = Path(self._temp_data_dir.name) / 'config.ini'
config.touch()
config.write_text(self._get_config())
self._steemd_process = Popen([
str(self._steemd_bin),
'--data-dir="' + str(self._temp_data_dir.name) + '"'
],
stdout=self.steemd_out,
stderr=self.steemd_err)
self._steemd_process.poll()
sleep(5)
if (not self._steemd_process.returncode):
self._rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '')
else:
raise Exception("steemd did not start properly...")
def __init__(self, config):
""" Initialize configuration
"""
available_features = dir(config)
if ("wallet_host" in available_features and
"wallet_port" in available_features):
self.wallet_host = config.wallet_host
self.wallet_port = config.wallet_port
if ("wallet_user" in available_features and
"wallet_password" in available_features):
self.wallet_user = config.wallet_user
self.wallet_password = config.wallet_password
self.rpc = SteemWalletRPC(self.wallet_host,
self.wallet_port,
self.wallet_user,
self.wallet_password)
# Make a reference to 'wallet'
self.wallet = self.rpc
# Connect to Witness Node
if "witness_url" in available_features:
self.witness_url = config.witness_url
if ("witness_user" in available_features):
self.witness_user = config.witness_user
if ("witness_password" in available_features):
self.witness_password = config.witness_password
self.ws = SteemNodeRPC(self.witness_url,
self.witness_user,
self.witness_password)
# Make a reference to 'node'
self.node = self.ws
def account_meta(account_name):
rpc = SteemNodeRPC("wss://steemit.com/wspa")
#try:
#account_meta = json.loads(rpc.get_account(account_name)['json_metadata'])
#account_url = account_meta['url']
last_tx, curr_tx = -1, -1
account_data, account_block, account_titles, account_links = [], [], [], []
while last_tx == curr_tx:
curr_tx = curr_tx + 1
account_block = rpc.get_account_history(account_name, curr_tx, 0)
last_tx = account_block[0][0]
if account_block[0][1]['op'][0] == 'comment' and account_block[0][1][
'op'][1]['author'] == account_name:
#account_data.append(account_block)
account_links.append(account_block[0][1]['op'][1]['permlink'])
account_titles.append(account_block[0][1]['op'][1]['title'])
print(str(account_titles[-1]) + ' - found your post!')
accountset = set(account_titles)
return (accountset)
def stream_blocks(server, block_nums, start, end):
"""Stream blocks from steemd in JSON format
\b
Which Steemd:
\b
1. CLI "--server" option if provided
2. ENV var "WEBSOCKET_URL" if provided
3. Default: "wss://steemit.com/wspa"
\b
Which Blocks To Output:
\b
- Stream blocks beginning with current block by omitting --start, --end, and BLOCKS
- Fetch a range of blocks using --start and/or --end
- Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN)
Where To Output Blocks:
\b
2. ENV var "BLOCKS_OUT" if provided
3. Default: STDOUT
"""
# Setup steemd source
rpc = SteemNodeRPC(server)
with click.open_file('-', 'w', encoding='utf8') as f:
if block_nums:
blocks = _stream_blocks(rpc, block_nums)
elif start and end:
blocks = _stream_blocks(rpc, range(start, end))
else:
blocks = rpc.block_stream(start)
json_blocks = map(json.dumps, blocks)
for block in json_blocks:
click.echo(block, file=f)
from pprint import pprint
import time
"""
Connection Parameters to steemd daemon.
Start the steemd daemon with the rpc-endpoint parameter:
./programs/steemd/steemd --rpc-endpoint=127.0.0.1:8092
This opens up a RPC port (e.g. at 8092). Currently, authentication
is not yet available, thus, we recommend to restrict access to
localhost. Later we will allow authentication via username and
passpword (both empty now).
"""
rpc = SteemNodeRPC("ws://localhost:8090", "", "")
"""
Last Block that you have process in your backend.
Processing will continue at `last_block + 1`
"""
last_block = 160900
"""
Deposit account name to monitor
"""
watch_account = "world"
def process_block(block, blockid):
"""
This call processes a block which can carry many transactions
from datetime import datetime, timedelta
from steemapi.steemnoderpc import SteemNodeRPC
from piston.steem import Post
from pymongo import MongoClient
from pprint import pprint
import json
import time
import sys
import os
rpc = SteemNodeRPC("ws://" + os.environ['steemnode'], "", "")
mongo = MongoClient("mongodb://mongo")
db = mongo.steemdb
init = db.status.find_one({'_id': 'height'})
if (init):
last_block = init['value']
else:
last_block = 1
# ------------
# For development:
#
# If you're looking for a faster way to sync the data and get started,
# uncomment this line with a more recent block, and the chain will start
# to sync from that point onwards. Great for a development environment
# where you want some data but don't want to sync the entire blockchain.
# ------------
# last_block = 4155048
from datetime import datetime
from steemapi.steemnoderpc import SteemNodeRPC
from piston.steem import Post
from pymongo import MongoClient
from pprint import pprint
import collections
import time
import sys
import os
from apscheduler.schedulers.background import BackgroundScheduler
# rpc = SteemNodeRPC(host, "", "", ['follow_api'])
rpc = SteemNodeRPC("ws://" + os.environ['steemnode'],
"",
"",
apis=["follow", "database"])
mongo = MongoClient("mongodb://mongo")
db = mongo.steemdb
mvest_per_account = {}
def load_accounts():
pprint("SteemDB - Loading mvest per account")
for account in db.account.find():
if "name" in account.keys():
mvest_per_account.update(
{account['name']: account['vesting_shares']})
# thanks @furion for the tip
from steemapi.steemnoderpc import SteemNodeRPC
rpc = SteemNodeRPC('ws://node.steem.ws')
import csv
allauthors = rpc.lookup_accounts('', 1000000)
postdict = dict()
repdict = dict()
rewdict = dict()
# counting author posts and adding them to the postdict dictionary
# retrieving authors with more than 700 posts (blogposts and comments)
for author in allauthors:
postcount = rpc.get_account(author)['post_count']
if postcount > 700:
postdict[author] = postcount
# retrieving reputation and post rewards for the authors in postdict
for author2 in postdict.keys():
repcount = rpc.get_account(author2)['reputation']
rewcount = rpc.get_account(author2)['posting_rewards']
repdict[author2] = repcount
rewdict[author2] = rewcount
# indexing results for posts
writefile1 = open('productivesteemians-posts.csv', 'w', newline='')
writer1 = csv.writer(writefile1)
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs,
max_threads):
# pylint: disable=too-many-locals, too-many-statements
rpc = SimpleSteemAPIClient(steemd_http_url)
engine_config = configure_engine(database_url)
db_name = engine_config.url.database
db_user_name = engine_config.url.username
Session.configure(bind=engine_config.engine)
session = Session()
# [1/7] confirm db connectivity
task_message = fmt_task_message(
'Confirm database connectivity',
emoji_code_point=u'\U0001F4DE',
counter=1)
click.echo(task_message)
url, table_count = test_connection(database_url)
if url:
success_msg = fmt_success_message(
'connected to %s and found %s tables', url.__repr__(), table_count)
click.echo(success_msg)
if not url:
raise Exception('Unable to connect to database')
del url
del table_count
# [2/7] kill existing db threads
task_message = fmt_task_message(
'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2)
click.echo(task_message)
all_procs, killed_procs = kill_db_processes(database_url, db_name,
db_user_name)
if len(killed_procs) > 0:
success_msg = fmt_success_message('killed %s processes',
len(killed_procs))
click.echo(success_msg)
del all_procs
del killed_procs
# [3/7] init db if required
task_message = fmt_task_message(
'Initialising db if required',
emoji_code_point=u'\U0001F50C',
counter=3)
click.echo(task_message)
init_tables(database_url, Base.metadata)
# [4/7] find last irreversible block
last_chain_block = rpc.last_irreversible_block_num()
task_message = fmt_task_message(
'Finding highest blockchain block',
emoji_code_point='\U0001F50E',
counter=4)
click.echo(task_message)
success_msg = fmt_success_message(
'learned highest irreversible block is %s', last_chain_block)
click.echo(success_msg)
# [5/7] get missing block_nums
task_message = fmt_task_message(
'Finding blocks missing from db',
emoji_code_point=u'\U0001F52D',
counter=5)
click.echo(task_message)
missing_block_nums_gen = Block.get_missing_block_num_iterator(
session, last_chain_block, chunksize=100000)
with click.progressbar(
missing_block_nums_gen,
label='Finding missing block_nums',
color=True,
show_eta=False,
show_percent=False,
empty_char='░',
fill_char='█',
show_pos=True,
bar_template='%(bar)s %(info)s') as pbar:
all_missing_block_nums = []
for missing_gen in pbar:
all_missing_block_nums.extend(missing_gen())
success_msg = fmt_success_message('found %s missing blocks',
len(all_missing_block_nums))
click.echo(success_msg)
del missing_block_nums_gen
del pbar
session.invalidate()
# [6/7] adding missing blocks
task_message = fmt_task_message(
'Adding missing blocks to db, this may take a while',
emoji_code_point=u'\U0001F4DD',
counter=6)
click.echo(task_message)
max_workers = max_procs or os.cpu_count() or 1
chunksize = len(all_missing_block_nums) // max_workers
if chunksize <= 0:
chunksize = 1
map_func = partial(
block_adder_process_worker,
database_url,
steemd_http_url,
max_threads=max_threads)
chunks = chunkify(all_missing_block_nums, 10000)
with concurrent.futures.ProcessPoolExecutor(
max_workers=max_workers) as executor:
executor.map(map_func, chunks , chunksize=1)
success_msg = fmt_success_message('added missing blocks')
click.echo(success_msg)
del all_missing_block_nums
# [7/7] stream blocks
task_message = fmt_task_message(
'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7)
click.echo(task_message)
highest_db_block = Block.highest_block(session)
ws_rpc = SteemNodeRPC(steemd_websocket_url)
blocks = ws_rpc.block_stream(highest_db_block)
add_blocks(blocks, session)
def __call__(self, form, field):
from steemapi.steemnoderpc import SteemNodeRPC
try:
SteemNodeRPC(field.data, num_retries=0)
except:
raise ValidationError(self.message)
def __init__(self):
self.rpc = SteemNodeRPC("wss://node.steem.ws", "", "", apis=["follow"])
from steemapi.steemnoderpc import SteemNodeRPC
from pprint import pprint
rpc = SteemNodeRPC("wss://steemit.com/ws")
for a in rpc.stream("comment", start=1893850):
pprint(a)
