Пример #1
0
   def __enter__( self ):
      self._steemd_lock.acquire()

      # Setup temp directory to use as the data directory for this
      self._temp_data_dir = TemporaryDirectory()

      for child in self._data_dir.iterdir():
         if( child.is_dir() ):
            copytree( str( child ), str( self._temp_data_dir.name ) + '/' + child.name )

      db_version = Path( self._data_dir.name ) / 'db_version'
      if( db_version.exists() and not db_version.is_dir() ):
         copy2( str( db_version ), str( self._temp_data_dir.name ) + '/db_version' )

      config = Path( self._temp_data_dir.name ) / 'config.ini'
      config.touch()
      config.write_text( self._get_config() )

      steemd = [ str( self._steemd_bin ), '--data-dir=' + str( self._temp_data_dir.name ) ]
      steemd.extend( self._args )

      self._steemd_process = Popen( steemd, stdout=self.steemd_out, stderr=self.steemd_err )
      self._steemd_process.poll()
      sleep( 5 )
      if( not self._steemd_process.returncode ):
         self._rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' )
      else:
         raise Exception( "steemd did not start properly..." )
Пример #2
0
def run_steemd_tests( debug_node ):
   from steemapi.steemnoderpc import SteemNodeRPC

   try:
      print( 'Replaying blocks...', )
      sys.stdout.flush()
      total_blocks = 0
      while( total_blocks % 100000 == 0 ):
         total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True )
         print( 'Blocks Replayed: ' + str( total_blocks ) )
         sys.stdout.flush()

      print( "Triggering payouts" )
      sys.stdout.flush()
      debug_node.debug_generate_blocks_until( 1467590400 - 3 )
      rpc = SteemNodeRPC( 'ws://127.0.0.1:8095', '', '' )
      ret = rpc.lookup_accounts( '', str( 0xFFFFFFFF ) )
      account_balances = {}
      for acc_name in ret:
         acc = rpc.get_accounts( [ acc_name ] )
         steemd = float( acc[0][ 'balance' ].split( ' ' )[0] )
         sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] )
         vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] )
         account_balances[ acc_name ] = ( steemd, sbd, vests )

      debug_node.debug_generate_blocks( 1 )

      account_rewards = {}
      for acc_name, bal in account_balances.items():
         acc = rpc.get_accounts( [ acc_name ] )
         steemd = float( acc[0][ 'balance' ].split( ' ' )[0] ) - bal[0]
         sbd = float( acc[0][ 'sbd_balance' ].split( ' ' )[0] ) - bal[1]
         vests = float( acc[0][ 'vesting_shares' ].split( ' ' )[0] ) - bal[2]
         account_rewards[ acc_name ] = ( steemd, sbd, vests )

      print( "Generating blocks to verify nothing broke" )
      assert( debug_node.debug_generate_blocks( 10 ) == 10 )

      '''
      print( "Done!" )
      print( "Getting comment dump:" )
      sys.stdout.flush()

      ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) );

      print( 'author, url, total_payout_value, abs_rshares, num_active_votes' )

      for comment in ret:
         print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] )
      '''

      print( "Printing account reward dump:" )
      print( "account, steem, sbd, vests" )
      for acc_name, rew in account_rewards.items():
         print( acc_name + ', ' + str( rew[0] ) + ' STEEM, ' + str( rew[1] ) + ' SBD, ' + str( rew[2] ) + ' VESTS' )

   except ValueError as val_err:
      print( str( val_err ) )
Пример #3
0
def run_steemd_tests(debug_node):
    from steemapi.steemnoderpc import SteemNodeRPC

    try:
        print('Replaying blocks...', )
        sys.stdout.flush()
        total_blocks = 0
        while (total_blocks % 100000 == 0):
            total_blocks += debug_node.debug_push_blocks(
                100000, skip_validate_invariants=True)
            print('Blocks Replayed: ' + str(total_blocks))
            sys.stdout.flush()

        print("Triggering payouts")
        sys.stdout.flush()
        debug_node.debug_generate_blocks_until(1467590400 - 3)
        rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '')
        ret = rpc.lookup_accounts('', str(0xFFFFFFFF))

        debug_node.debug_generate_blocks(1)

        print("Generating blocks to verify nothing broke")
        assert (debug_node.debug_generate_blocks(10) == 10)

        account_rewards = {}
        vote_count = {}
        for acc_name in ret:
            acc = rpc.get_accounts([acc_name])
            #print( acc_name + ',' + acc[0][ 'curation_rewards' ] )
            account_rewards[acc_name] = float(
                acc[0]['curation_rewards'].split(' ')[0])
            vote_count[acc_name] = int(acc[0]['lifetime_vote_count'])
        '''
      print( "Done!" )
      print( "Getting comment dump:" )
      sys.stdout.flush()

      ret = rpc.get_discussions_by_cashout_time( '', '', str( 0xFFFFFFFF ) );

      print( 'author, url, total_payout_value, abs_rshares, num_active_votes' )

      for comment in ret:
         print( comment[ 'author' ] + ', ' + comment[ 'url' ] + ', ' + comment[ 'total_payout_value' ] + ', ' + comment[ 'cashout_time' ] )

      '''
        print("Printing account reward dump:")
        sorted_rewards = sorted(account_rewards.items(),
                                key=operator.itemgetter(1))
        print("account, curation_steem")
        for rew in sorted_rewards:
            print(rew[0] + ', ' + str(rew[1]) + ', ' + str(vote_count[rew[0]]))

    except ValueError as val_err:
        print(str(val_err))
Пример #4
0
 def validate(self):
     from steemapi.steemnoderpc import SteemNodeRPC
     if not Form.validate(self):
         return False
     try:
         SteemNodeRPC(self.node.data,
                      user=self.rpcuser.data,
                      password=self.rpcpass.data,
                      num_retries=0)
     except:
         self.node.errors.append('Unable to connect')
         return False
     return True
Пример #5
0
def determine_user_index(account_name):
    #rpc = SteemNodeRPC("wss://steemit.com/wspa")

    try:
        try:
            [h_index, post_titles, sp_payouts
             ] = hindex_dict[account_name][str(datetime.date.today())]
            publish_date = str(datetime.date.today())
        except:
            print('Could not find in dictionary: ' + account_name + ', on: ' +
                  str(datetime.date.today()))
            rpc = SteemNodeRPC("ws://*****:*****@' + account_name + '/' + p for p in post_titles])

    return render_template('index.html',
                           name=account_name,
                           body=body,
                           post_list=post_list)
Пример #6
0
def determine_quick_user_index(account_name):
    rpc = SteemNodeRPC("wss://steemit.com/wspa")

    #rpc = SteemNodeRPC("ws://localhost:8090")
    post_payouts, post_votes, post_titles, post_links = author_rewards_quick(
        rpc, account_name)

    sp_payouts = rewards_conversion(post_payouts)

    h_index = 0
    for pos, val in enumerate(sp_payouts):
        if val > pos + 1:
            h_index = pos + 1
        else:
            break

    h2_index = 0
    for pos, val in enumerate(sorted(post_votes, reverse=True)):
        if val > pos + 1:
            h2_index = pos + 1
        else:
            break

    publish_date = str(datetime.date.today())

    # return('V-index (reward value): ' + str(h_index) + '<br>P-index (popular votes): ' + str(h2_index) + '<hr>Most popular post: <a href=\"http://steemit.com' + post_links[0] + '\">' + post_titles[0] + '</a>, at a value of ' + str(sp_payouts[0]) + ' Steem Power.')

    body = 'V-index (reward value): ' + str(
        h_index) + '<br><br>P-index (popular votes): ' + str(h2_index)
    post_list = generate_table(sp_payouts, post_votes, post_titles, post_links)

    if post_list == "":
        return render_template('index.html',
                               name=account_name,
                               body=('<h1>Something went wrong.  Has ' +
                                     account_name + ' ever posted?</h1>'))

    return render_template('index.html',
                           name=account_name,
                           body=body,
                           post_list=post_list)
Пример #7
0
    def __enter__(self):
        self._steemd_lock.acquire()

        # Setup temp directory to use as the data directory for this
        self._temp_data_dir = TemporaryDirectory()
        config = Path(self._temp_data_dir.name) / 'config.ini'
        config.touch()
        config.write_text(self._get_config())

        self._steemd_process = Popen([
            str(self._steemd_bin),
            '--data-dir="' + str(self._temp_data_dir.name) + '"'
        ],
                                     stdout=self.steemd_out,
                                     stderr=self.steemd_err)
        self._steemd_process.poll()
        sleep(5)
        if (not self._steemd_process.returncode):
            self._rpc = SteemNodeRPC('ws://127.0.0.1:8095', '', '')
        else:
            raise Exception("steemd did not start properly...")
Пример #8
0
    def __init__(self, config):
        """ Initialize configuration
        """
        available_features = dir(config)

        if ("wallet_host" in available_features and
                "wallet_port" in available_features):
            self.wallet_host = config.wallet_host
            self.wallet_port = config.wallet_port

            if ("wallet_user" in available_features and
                    "wallet_password" in available_features):
                self.wallet_user = config.wallet_user
                self.wallet_password = config.wallet_password

            self.rpc = SteemWalletRPC(self.wallet_host,
                                      self.wallet_port,
                                      self.wallet_user,
                                      self.wallet_password)

            # Make a reference to 'wallet'
            self.wallet = self.rpc

        # Connect to Witness Node
        if "witness_url" in available_features:
            self.witness_url = config.witness_url

            if ("witness_user" in available_features):
                self.witness_user = config.witness_user

            if ("witness_password" in available_features):
                self.witness_password = config.witness_password

            self.ws = SteemNodeRPC(self.witness_url,
                                   self.witness_user,
                                   self.witness_password)

            # Make a reference to 'node'
            self.node = self.ws
Пример #9
0
def account_meta(account_name):
    rpc = SteemNodeRPC("wss://steemit.com/wspa")

    #try:
    #account_meta = json.loads(rpc.get_account(account_name)['json_metadata'])

    #account_url = account_meta['url']

    last_tx, curr_tx = -1, -1
    account_data, account_block, account_titles, account_links = [], [], [], []
    while last_tx == curr_tx:
        curr_tx = curr_tx + 1
        account_block = rpc.get_account_history(account_name, curr_tx, 0)
        last_tx = account_block[0][0]
        if account_block[0][1]['op'][0] == 'comment' and account_block[0][1][
                'op'][1]['author'] == account_name:
            #account_data.append(account_block)
            account_links.append(account_block[0][1]['op'][1]['permlink'])
            account_titles.append(account_block[0][1]['op'][1]['title'])
            print(str(account_titles[-1]) + ' - found your post!')

    accountset = set(account_titles)
    return (accountset)
Пример #10
0
def stream_blocks(server, block_nums, start, end):
    """Stream blocks from steemd in JSON format

    \b
    Which Steemd:
    \b
    1. CLI "--server" option if provided
    2. ENV var "WEBSOCKET_URL" if provided
    3. Default: "wss://steemit.com/wspa"

    \b
    Which Blocks To Output:
    \b
    - Stream blocks beginning with current block by omitting --start, --end, and BLOCKS
    - Fetch a range of blocks using --start and/or --end
    - Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN)

    Where To Output Blocks:

    \b
    2. ENV var "BLOCKS_OUT" if provided
    3. Default: STDOUT
    """
    # Setup steemd source
    rpc = SteemNodeRPC(server)
    with click.open_file('-', 'w', encoding='utf8') as f:
        if block_nums:
            blocks = _stream_blocks(rpc, block_nums)
        elif start and end:
            blocks = _stream_blocks(rpc, range(start, end))
        else:
            blocks = rpc.block_stream(start)

        json_blocks = map(json.dumps, blocks)

        for block in json_blocks:
            click.echo(block, file=f)
Пример #11
0
from pprint import pprint
import time
"""
   Connection Parameters to steemd daemon.

   Start the steemd daemon with the rpc-endpoint parameter:

      ./programs/steemd/steemd --rpc-endpoint=127.0.0.1:8092

    This opens up a RPC port (e.g. at 8092). Currently, authentication
    is not yet available, thus, we recommend to restrict access to
    localhost. Later we will allow authentication via username and
    passpword (both empty now).

"""
rpc = SteemNodeRPC("ws://localhost:8090", "", "")
"""
    Last Block that you have process in your backend.
    Processing will continue at `last_block + 1`
"""
last_block = 160900
"""
    Deposit account name to monitor
"""
watch_account = "world"


def process_block(block, blockid):
    """
        This call processes a block which can carry many transactions
Пример #12
0
from datetime import datetime, timedelta
from steemapi.steemnoderpc import SteemNodeRPC
from piston.steem import Post
from pymongo import MongoClient
from pprint import pprint
import json
import time
import sys
import os

rpc = SteemNodeRPC("ws://" + os.environ['steemnode'], "", "")
mongo = MongoClient("mongodb://mongo")
db = mongo.steemdb

init = db.status.find_one({'_id': 'height'})
if (init):
    last_block = init['value']
else:
    last_block = 1

# ------------
# For development:
#
# If you're looking for a faster way to sync the data and get started,
# uncomment this line with a more recent block, and the chain will start
# to sync from that point onwards. Great for a development environment
# where you want some data but don't want to sync the entire blockchain.
# ------------

# last_block = 4155048
Пример #13
0
from datetime import datetime
from steemapi.steemnoderpc import SteemNodeRPC
from piston.steem import Post
from pymongo import MongoClient
from pprint import pprint
import collections
import time
import sys
import os

from apscheduler.schedulers.background import BackgroundScheduler

# rpc = SteemNodeRPC(host, "", "", ['follow_api'])

rpc = SteemNodeRPC("ws://" + os.environ['steemnode'],
                   "",
                   "",
                   apis=["follow", "database"])
mongo = MongoClient("mongodb://mongo")
db = mongo.steemdb

mvest_per_account = {}


def load_accounts():
    pprint("SteemDB - Loading mvest per account")
    for account in db.account.find():
        if "name" in account.keys():
            mvest_per_account.update(
                {account['name']: account['vesting_shares']})

# thanks @furion for the tip

from steemapi.steemnoderpc import SteemNodeRPC
rpc = SteemNodeRPC('ws://node.steem.ws')
import csv

allauthors = rpc.lookup_accounts('', 1000000)
postdict = dict()
repdict = dict()
rewdict = dict()

# counting author posts and adding them to the postdict dictionary
# retrieving authors with more than 700 posts (blogposts and comments)

for author in allauthors:
    postcount = rpc.get_account(author)['post_count']
    if postcount > 700:
        postdict[author] = postcount

# retrieving reputation and post rewards for the authors in postdict

for author2 in postdict.keys():
    repcount = rpc.get_account(author2)['reputation']
    rewcount = rpc.get_account(author2)['posting_rewards']
    repdict[author2] = repcount
    rewdict[author2] = rewcount

# indexing results for posts

writefile1 = open('productivesteemians-posts.csv', 'w', newline='')
writer1 = csv.writer(writefile1)
Пример #15
0
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs,
              max_threads):
    # pylint: disable=too-many-locals, too-many-statements
    rpc = SimpleSteemAPIClient(steemd_http_url)
    engine_config = configure_engine(database_url)

    db_name = engine_config.url.database
    db_user_name = engine_config.url.username

    Session.configure(bind=engine_config.engine)
    session = Session()

    # [1/7] confirm db connectivity
    task_message = fmt_task_message(
        'Confirm database connectivity',
        emoji_code_point=u'\U0001F4DE',
        counter=1)
    click.echo(task_message)

    url, table_count = test_connection(database_url)
    if url:
        success_msg = fmt_success_message(
            'connected to %s and found %s tables', url.__repr__(), table_count)
        click.echo(success_msg)

    if not url:
        raise Exception('Unable to connect to database')
    del url
    del table_count

    # [2/7] kill existing db threads
    task_message = fmt_task_message(
        'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2)
    click.echo(task_message)
    all_procs, killed_procs = kill_db_processes(database_url, db_name,
                                                db_user_name)
    if len(killed_procs) > 0:
        success_msg = fmt_success_message('killed %s processes',
                                          len(killed_procs))
        click.echo(success_msg)
    del all_procs
    del killed_procs

    # [3/7] init db if required
    task_message = fmt_task_message(
        'Initialising db if required',
        emoji_code_point=u'\U0001F50C',
        counter=3)
    click.echo(task_message)
    init_tables(database_url, Base.metadata)

    # [4/7] find last irreversible block
    last_chain_block = rpc.last_irreversible_block_num()
    task_message = fmt_task_message(
        'Finding highest blockchain block',
        emoji_code_point='\U0001F50E',
        counter=4)
    click.echo(task_message)

    success_msg = fmt_success_message(
        'learned highest irreversible block is %s', last_chain_block)
    click.echo(success_msg)

    # [5/7] get missing block_nums
    task_message = fmt_task_message(
        'Finding blocks missing from db',
        emoji_code_point=u'\U0001F52D',
        counter=5)
    click.echo(task_message)
    missing_block_nums_gen = Block.get_missing_block_num_iterator(
        session, last_chain_block, chunksize=100000)

    with click.progressbar(
            missing_block_nums_gen,
            label='Finding missing block_nums',
            color=True,
            show_eta=False,
            show_percent=False,
            empty_char='░',
            fill_char='█',
            show_pos=True,
            bar_template='%(bar)s  %(info)s') as pbar:
        all_missing_block_nums = []
        for missing_gen in pbar:
            all_missing_block_nums.extend(missing_gen())

    success_msg = fmt_success_message('found %s missing blocks',
                                      len(all_missing_block_nums))
    click.echo(success_msg)
    del missing_block_nums_gen
    del pbar
    session.invalidate()

    # [6/7] adding missing blocks
    task_message = fmt_task_message(
        'Adding missing blocks to db, this may take a while',
        emoji_code_point=u'\U0001F4DD',
        counter=6)
    click.echo(task_message)

    max_workers = max_procs or os.cpu_count() or 1

    chunksize = len(all_missing_block_nums) // max_workers
    if chunksize <= 0:
        chunksize = 1

    map_func = partial(
        block_adder_process_worker,
        database_url,
        steemd_http_url,
        max_threads=max_threads)

    chunks = chunkify(all_missing_block_nums, 10000)

    with concurrent.futures.ProcessPoolExecutor(
            max_workers=max_workers) as executor:
        executor.map(map_func, chunks , chunksize=1)

    success_msg = fmt_success_message('added missing blocks')
    click.echo(success_msg)
    del all_missing_block_nums

    # [7/7] stream blocks
    task_message = fmt_task_message(
        'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7)
    click.echo(task_message)

    highest_db_block = Block.highest_block(session)
    ws_rpc = SteemNodeRPC(steemd_websocket_url)
    blocks = ws_rpc.block_stream(highest_db_block)
    add_blocks(blocks, session)
Пример #16
0
 def __call__(self, form, field):
     from steemapi.steemnoderpc import SteemNodeRPC
     try:
         SteemNodeRPC(field.data, num_retries=0)
     except:
         raise ValidationError(self.message)
Пример #17
0
 def __init__(self):
     self.rpc = SteemNodeRPC("wss://node.steem.ws", "", "", apis=["follow"])
Пример #18
0
from steemapi.steemnoderpc import SteemNodeRPC
from pprint import pprint

rpc = SteemNodeRPC("wss://steemit.com/ws")

for a in rpc.stream("comment", start=1893850):
    pprint(a)