def __init__(self, **kwargs): super().__init__(**kwargs) # init blockchain steemd_url = self.yo_app.config.steemd_url self.steemd = steem.steemd.Steemd(nodes=[steemd_url]) self.blockchain = Blockchain(steemd_instance=self.steemd) self.start_block = self.yo_app.config.steemd_start_block self.last_block_num_handled = 0 # init ops handlers self.op_map = defaultdict(list) self.op_map.update({ 'vote': [self.handle_vote], 'account_update': [self.handle_account_update], 'transfer': [self.handle_send, self.handle_receive], 'custom_json': [self.handle_follow, self.handle_resteem], 'withdraw_vesting': [self.handle_power_down], 'comment': [self.handle_mention, self.handle_comment] }) start_block = self.get_start_block() self.ops_func = self.blockchain.stream_from( start_block=start_block, batch_operations=True) self.ops = lambda: self.execute_sync(next, self.ops_func)
def run(self): while True: try: b = Blockchain(Steem(nodes=app.config['STEEM_NODES'])) log('Using Steem API node(s): ' + str(app.config['STEEM_NODES'])) log('Blockchain head is ' + str(steem.head_block_number)) # start from max block present in table post = db.session.query(Post).order_by(Post.id.desc()).first() if post: self.start_block = post.block_number log('Starting streaming (in catch up) from block: ' + str(self.start_block)) else: self.start_block = self.start_block log('Starting streaming from (specified) block: ' + str(self.start_block)) for blockcount, block in enumerate(b.stream_from(start_block=self.start_block, full_blocks=True)): block_number = self.start_block + blockcount if (self.start_block + blockcount) % 20 == 0: log('Read Block:' + str(block_number)) try: add_block_content_to_db(block) except Exception as e: log('ERROR: Problem adding block to database.') log(str(e)) time.sleep(10) break except Exception as e: log('ERROR: Problem collecting raw blocks from ' + str(app.config['STEEM_NODES'])) log(str(e)) time.sleep(10)
def scrape_operations(mongo): """Fetch all operations (including virtual) from last known block forward.""" indexer = Indexer(mongo) last_block = indexer.get_checkpoint('operations') log.info('\n> Fetching operations, starting with block %d...' % last_block) blockchain = Blockchain(mode="irreversible") history = blockchain.history( start_block=last_block, ) for operation in history: # insert operation with suppress(DuplicateKeyError): transform = compose(strip_dot_from_keys, json_expand, typify) mongo.Operations.insert_one(transform(operation)) # if this is a new block, checkpoint it, and schedule batch processing if operation['block_num'] != last_block: last_block = operation['block_num'] indexer.set_checkpoint('operations', last_block - 1) if last_block % 10 == 0: log.info("Checkpoint: %s (%s)" % ( last_block, blockchain.steem.hostname ))
def stream(): REGEX = "(?<=^|(?<=[^a-zA-Z0-9-_\.]))!([A-Za-z]+[A-Za-z0-9]+)" # REGEX code for '!' marker blockchain = Blockchain() # blockchain instance stream = map( Post, blockchain.stream(filter_by=['comment'])) # stream comments only print('\033[37mSearching...') for post in stream: curator = post['author'] p_parent = post['parent_permlink'] a_parent = post['parent_author'] f = regex.findall(REGEX, post['body']) # REGEX searches for !'flag' # filters if post.is_comment(): if curator in curators: if flag in f: if p_parent in log: continue else: # start action thread acaoThread = threading.Thread(target=action, args=( a_parent, p_parent, curator, )) acaoThread.start()
def validate_operations(mongo): """ Scan latest N blocks in the database and validate its operations. """ blockchain = Blockchain(mode="irreversible") highest_block = mongo.Operations.find_one({}, sort=[('block_num', -1) ])['block_num'] lowest_block = max(1, highest_block - 250_000) for block_num in range(highest_block, lowest_block, -1): if block_num % 100 == 0: log.info('Validating block #%s' % block_num) block = list( blockchain.stream(start_block=block_num, end_block=block_num)) # remove all invalid or changed operations conditions = { 'block_num': block_num, '_id': { '$nin': [x['_id'] for x in block] } } mongo.Operations.delete_many(conditions) # insert any missing operations for op in block: with suppress(DuplicateKeyError): transform = compose(strip_dot_from_keys, json_expand, typify) mongo.Operations.insert_one(transform(op)) # re-process comments (does not re-add deleted posts) for comment in (x for x in block if x['type'] == 'comment'): upsert_comment(mongo, '%s/%s' % (comment['author'], comment['permlink']))
def get_transactionid(username): time.sleep(6) found = False count = 0 while found == False and count < 5: count = count +1 print ("Attempt to get an ID number: " +str(count)) b = Blockchain() end_block = steem.head_block_number start_block = end_block -100 for block in b.stream_from(start_block=start_block, end_block=end_block, full_blocks=True): transactions = block['transactions'] for transaction in transactions: if (transaction['operations'][0][0] == "custom_json"): if (transaction['operations'][0][1]['required_posting_auths'][0]) == user: print (transaction) if (transaction['operations'][0][1]['required_posting_auths'][0]) == user and (transaction['operations'][0][1]['id'])=="sm_find_match": print (transaction['transaction_id']) trxid = (transaction['transaction_id']) #check if this is a valid id r = requests.get('https://steemmonsters.com/battle/status?id=' +str(trxid)) data = r.json() try: status = data['status'] print ("ID Status:" +str(status)) if int(status) == 1: found = True return (trxid) except: found = False return
def run(): username = os.environ.get("STEEM_NAME") blockchain = Blockchain() stream = map(Post, blockchain.stream(filter_by=["comment"])) upvote_list = json.load(open('upvote_list.json')) wif = upvote_list["voters"][username]["wif"] steem = Steem(wif=wif) upvoted = {} date = int(time.strftime("%d")) hour = int(time.strftime("%-H")) print("Entering blockchain stream!") while True: try: for post in stream: if int(time.strftime("%d")) is not date: upvoted = {} date = int(time.strftime("%d")) if int(time.strftime("%-H")) is not hour: upvote_list = json.load(open('upvote_list.json')) hour = int(time.strftime("%-H")) if valid_post(post, upvote_list, upvoted): try: author = post["author"] schedule_upvote(upvote_list, username, author) print("Upvoted {}".format(author)) upvoted[author] += 1 except Exception as error: print(repr(error)) continue except Exception as error: #print(repr(error)) continue
def run(): blockchain = Blockchain() head_block = blockchain.get_current_block_num() start_block = head_block - (int(sys.argv[1])) stream = blockchain.stream_from(start_block=start_block) block_count = int(sys.argv[1]) current_block = start_block stats = {} operations = 0 counter = 0 print("Starting from block {} for {} blocks\n".format( start_block, block_count)) for post in stream: if post['block'] != current_block: counter += 1 print("Block {}/{} {:.2f}%".format(counter, block_count, counter / block_count * 100)) current_block = post['block'] elif post['block'] == start_block + block_count: break operation = post['op'][0] if operation not in stats: stats[operation] = 1 else: stats[operation] += 1 operations += 1 print("operations {}\n".format(operations)) for operation in stats: print(operation, stats[operation])
def main(): accounts = [] votes_db = pickledb.load('steem_account_votes.db', False) comments_db = pickledb.load('steem_account_comments.db', False) transfers_db = pickledb.load('steem_account_trasnfers.db', False) vesting_transfers_db = pickledb.load('steem_account_vesting_transfers.db', False) result_db = pickledb.load('result.db', False) steemd = Steem(STEEM_NODES) chain = Blockchain(steemd_instance = steemd, mode = 'head') logging.getLogger().setLevel(20) databases = {'account_create': accounts, 'vote': votes_db, 'comment': comments_db, 'transfer': transfers_db, 'transfer_to_vesting': vesting_transfers_db} fill(databases, chain, round((1497970800 - 1451606400) / 3), chain.info()['head_block_number']) votes_db.dump() comments_db.dump() transfers_db.dump() vesting_transfers_db.dump() analyze(databases, result_db) result_db.dump()
def run(user_json): username = "******" wif = os.environ.get("UNLOCK") steem = Steem(wif=wif) blockchain = Blockchain() stream = map(Post, blockchain.stream(filter_by=["comment"])) print("Checking posts on the blockchain!") while True: try: for post in stream: title = post["title"] author = post["author"] print(author) if author in user_json: # if valid_post(post, user_json): try: title = post["title"] author = post["author"] print("Upvoting post {} by {}!".format(title, author)) post.upvote(weight=user_json[author]["upvote_weight"], voter=username) except Exception as error: print(repr(error)) continue except Exception as error: print(repr(error)) continue
def reconfigure_node(self): self.node_index = (self.node_index + 1) % len(self.list) self.nodes = [self.list[self.node_index]] self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes) self.b = Blockchain(self.steem) self.queue.steem = self.steem print('New node: {}\n'.format(self.nodes))
def run(filename): b = Blockchain() # automatically resume from where we left off # previous + last + 1 start_block = get_previous_block_num(get_last_line(filename)) + 2 with open(filename, 'a+') as file: for block in b.stream_from(start_block=start_block, full_blocks=True): file.write(json.dumps(block, sort_keys=True) + '\n')
def run(): account = "sttest1" account = Account(account) wif = os.environ.get("UNLOCK") steem = Steem(wif=wif) blockchain = Blockchain() stream = map(Post, blockchain.stream(filter_by=["comment"])) upvote_list = json.load(open('upvote_list.json')) upvoted = {} upvote_queue = {} date = int(time.strftime("%d")) hour = int(time.strftime("%-H")) print("Entering blockchain stream!") while True: try: for post in stream: time_string = str(post.time_elapsed()) post_age = time_string.split(":") try: # Omit posts older than 10 min if int(post_age[0]) > 0 or int(post_age[1]) > 10: break except Exception as e: #print (repr(e)) break # start upvoting cycle at 100% voting power if account.voting_power() == 100: start_upvote_cycle(upvote_queue, account, steem) # check for new date if int(time.strftime("%d")) is not date: upvoted = {} date = int(time.strftime("%d")) #check for new hour if int(time.strftime("%-H")) is not hour: upvote_list = json.load(open('upvote_list.json')) hour = int(time.strftime("%-H")) # verify post and add to queue if valid_post(post, upvote_list, upvoted): try: author = post["author"] print ("\nAdding to queue{}\nPermlink: {}".format(author, post['identifier'])) if post['identifier'] not in upvote_queue: upvote_queue[post['identifier']] = upvote_list[author]["upvote_weight"] print ("Upvoted {} for {}%".format(author, upvote_list[author]["upvote_weight"])) upvoted[author] += 1 except Exception as error: print(repr(error)) continue except Exception as error: #print(repr(error)) continue
def __init__(self, authorsLiked, who, mainWin): self.authorsLiked = authorsLiked self.tagsLiked = ['polish'] self.who = who self.b = Blockchain() self.stream = self.b.stream() self.mainWin = mainWin
def __init__(self, block, block_count, operation): self.block = block self.end_block = block_count + self.block - 1 self.operation = operation self.nodes = ['https://rpc.buildteam.io', 'https://api.steemit.com', 'https://rpc.steemviz.com'] self.steem = Steem(nodes=self.nodes) self.b = Blockchain(self.steem) print('Booted\nConnected to: {}'.format(self.nodes[0]))
def run(): blockchain = Blockchain() id = blockchain.get_current_block_num() global current_id if id != current_id: block = blockchain.get_current_block() thread = threading.Thread(target=parse_next_block, args=(block, )) print("Start new thread", thread.name) thread.start() current_id = id threading.Timer(1, run).start()
def listen_steemd(): b = Blockchain() h = b.stream_from( start_block=db_last_block() + 1, full_blocks=True, ) for block in h: num = int(block['previous'][:8], base=16) + 1 print("[LIVE] Got block {} at {} with {} txs".format( num, block['timestamp'], len(block['transactions']))) process_blocks([block])
def run(tags): log.info("Follow mode activated", tags=tags) if tags is None or len(tags) < 1: raise ValueError("You must specify at least one tag") log.debug("initializing...") steem = Steem(keys=[cred.key]) account = Account(cred.id, steem) chain = Blockchain(steem) log.debug("ready", steem=steem, account=account, blockchain=chain) log.info("Gathering our following list...") following = account.get_following() pending = [] log.info("Following list retrieved", count=len(following)) log.info("Watching for new posts...") while True: stream = map(Post, chain.stream(filter_by=['comment'])) try: for post in stream: count = len(pending) if count > 0: copy = list(pending) for i in range(count): if have_bandwidth(steem, account): user = copy[i] log.info("following user", user=user) steem.follow(user, account=cred.id) del pending[0] else: log.warn("Waiting for more bandwidth before following another user") break if post.is_main_post(): log.debug("found a top-level post", author=post.author, tags=post.tags) if post.author != cred.id: for tag in tags: if tag in post.tags: if post.author not in following: pending.append(post.author) following.append(post.author) break except PostDoesNotExist as e: log.debug("Post has vanished", exception=e) except RPCError as e: log.error("RPC problem while streaming posts", exception=e)
def __init__(self, block, account): self.block = block self.account = account self.nodes = [ 'https://rpc.steemviz.com', 'https://rpc.buildteam.io', 'https://api.steemit.com' ] self.steemPostingKey = os.environ.get('steemPostingKey') self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes) self.b = Blockchain(self.steem) self.queue = classes.Queue(self.steem, self.account) print('\nConnected to: {}'.format(self.nodes[0]))
def __init__(self, acount, number): self.tag = 'vote' self.account = account self.number = number self.nodes = [ 'https://api.steemit.com', 'https://rpc.buildteam.io', 'https://rpc.steemviz.com' ] self.steem = Steem(nodes=self.nodes) self.b = Blockchain(self.steem) self.timestamp = None self.block = self.b.get_current_block_num() print('Booted\nConnected to: {}'.format(self.nodes[0]))
def stream_blockchain(): blockchain = Blockchain() stream = map(Post, blockchain.stream(filter_by=['comment'])) while True: try: for post in stream: tags = post["tags"] if post.is_main_post() and "utopian-io" in tags: author = post["author"] title = post["title"] print("{} posted {}".format(author, title)) except Exception as error: print(repr(error)) continue
def stream(self, opNames, *args, **kwargs): warnings.warn( "The stream() call has been moved to `steem.blockchain.Blockchain.stream()`", DeprecationWarning) from steem.blockchain import Blockchain return Blockchain(mode=kwargs.get("mode", "irreversible")).stream( opNames, *args, **kwargs)
def scrape_blockchain(mongo): s = Steem() # see how far behind we are missing = list(range(last_block_num(mongo), s.last_irreversible_block_num)) # if we are far behind blockchain head # split work in chunks of 100 if len(missing) > 100: for batch in partition_all(100, missing): results = s.get_blocks(batch) insert_blocks(mongo, results) # otherwise continue as normal blockchain = Blockchain(mode="irreversible") hist = blockchain.stream_from(start_block=last_block_num(mongo), full_blocks=True) insert_blocks(mongo, hist)
def get_block_headers_between(start_datetime, end_datetime, steem): """ Returns block headers between two dates""" latest_block_num = Blockchain(steem).get_current_block_num() end_offset_num, _ = find_nearest_block_num(end_datetime, steem, latest_block_num) return get_block_headers_between_offset_start(start_datetime, end_datetime, steem=steem, end_offset_num=end_offset_num)
def main(): global steem global tnt_server global steem_space global followers_space global chain print('starting datafeed.py..') ws_connection = os.environ['STEEMIT_WEBSOCKET_CONNECTION'] print('Connecting to ', ws_connection) sys.stdout.flush() steem = Steem(ws_connection) chain = Blockchain(steem_instance=steem, mode='head') while True: try: print('Connecting to tarantool (datastore:3301)..') sys.stdout.flush() tnt_server = tarantool.connect('datastore', 3301) steem_space = tnt_server.space('steem') followers_space = tnt_server.space('followers') except Exception as e: print('Cannot connect to tarantool server', file=sys.stderr) print(str(e), file=sys.stderr) sys.stderr.flush() time.sleep(10) continue else: while True: run() print('[run] exited, continue..')
def main(): global tnt_server global steem_space global followers_space global chain print('starting datafeed.py..') sys.stdout.flush() chain = Blockchain(mode='head') while True: try: print('Connecting to tarantool (datastore:3301)..') sys.stdout.flush() tnt_server = tarantool.connect('datastore', 3301) steem_space = tnt_server.space('steem') followers_space = tnt_server.space('followers') except Exception as e: print('Cannot connect to tarantool server', file=sys.stderr) print(str(e), file=sys.stderr) sys.stderr.flush() time.sleep(10) continue else: while True: run() print('[run] exited, continue..')
def list_accounts(self, start=None, step=1000, limit=None, **kwargs): warnings.warn( "The list_accounts() call has been moved to `steem.blockchain.Blockchain.get_all_accounts()`", DeprecationWarning) from steem.blockchain import Blockchain return Blockchain( mode=kwargs.get("mode", "irreversible")).get_all_accounts( start=start, steps=step, **kwargs)
def run(): # upvote posts with 30% weight upvote_pct = 30 whoami = 'makerhacks' # stream comments as they are published on the blockchain # turn them into convenient Post objects while we're at it b = Blockchain() stream = map(Post, b.stream(filter_by=['comment'])) for post in stream: if post.json_metadata: mentions = post.json_metadata.get('users', []) # if post mentions more than 10 people its likely spam if mentions and len(mentions) < 10: post.upvote(weight=upvote_pct, voter=whoami)
def run(): steem = Steem() blockchain = Blockchain() stream = blockchain.stream(filter_by=["transfer"]) username = "******" while True: try: for transfer in stream: if transfer["to"] == username: url, permlink = transfer["memo"].split("@") if "https://steemit.com/" in url: steem.vote(f"@{permlink}", 100) except Exception as error: print(repr(error)) continue
class Steem_node(): def __init__(self, block, block_count): self.block = block self.end_block = block_count + self.block - 1 self.tag = 'transfer' self.nodes = [ 'https://api.steemit.com', 'https://rpc.buildteam.io', 'https://rpc.steemviz.com' ] self.steem = Steem(nodes=self.nodes) self.b = Blockchain(self.steem) print('Booted\nConnected to: {}'.format(self.nodes[0])) def process_transaction(self, index, block, operation): date = block['timestamp'] to = operation['to'] user = operation['from'] amount = operation['amount'] memo = operation['memo'] db.insert_selection(self.block, index, date, to, user, amount, memo) def run(self): run = 1 # run until end_block while run == 1: try: # stream full blocks starting at start_block stream = self.b.stream_from(start_block=self.block, end_block=self.end_block, full_blocks=True) # process each block indiviudally for block in stream: print('\nBlock: ', self.block) # keep track of transaction index inside block index = 0 # go over each transaction indivually, process if tag is # met and update index for transaction in block['transactions']: if transaction['operations'][0][0] == self.tag: self.process_transaction( index, block, transaction['operations'][0][1]) index += 1 # Check if current block equals the end_block, break if so # else update the current block if self.block == self.end_block: run = 0 else: self.block += 1 except Exception as e: print('Error:', e) continue
from steem.blockchain import Blockchain # parse the whole chain for event in Blockchain().replay(): print("Event: %s" % event['op_type']) print("Time: %s" % event['timestamp']) print("Body: %s\n" % event['op']) # parse only payments from specific datetime until now b = Blockchain() history = b.replay( start_block=b.get_block_from_time("2016-09-01T00:00:00"), end_block=b.get_current_block(), filter_by=['transfer'] ) for payment in history: print("@%s sent %s to @%s" % (payment['from'], payment['amount'], payment['to'])) # Output: # @victoriart sent 1.000 SBD to @null # @dude sent 5.095 STEEM to @bittrex # @devil sent 5.107 STEEM to @poloniex # @pinoytravel sent 0.010 SBD to @null # @aladdin sent 5.013 STEEM to @poloniex # @mrwang sent 31.211 STEEM to @blocktrades # @kodi sent 0.030 SBD to @steembingo