def run(self): while True: try: b = Blockchain(Steem(nodes=app.config['STEEM_NODES'])) log('Using Steem API node(s): ' + str(app.config['STEEM_NODES'])) log('Blockchain head is ' + str(steem.head_block_number)) # start from max block present in table post = db.session.query(Post).order_by(Post.id.desc()).first() if post: self.start_block = post.block_number log('Starting streaming (in catch up) from block: ' + str(self.start_block)) else: self.start_block = self.start_block log('Starting streaming from (specified) block: ' + str(self.start_block)) for blockcount, block in enumerate(b.stream_from(start_block=self.start_block, full_blocks=True)): block_number = self.start_block + blockcount if (self.start_block + blockcount) % 20 == 0: log('Read Block:' + str(block_number)) try: add_block_content_to_db(block) except Exception as e: log('ERROR: Problem adding block to database.') log(str(e)) time.sleep(10) break except Exception as e: log('ERROR: Problem collecting raw blocks from ' + str(app.config['STEEM_NODES'])) log(str(e)) time.sleep(10)
def run(filename): b = Blockchain() # automatically resume from where we left off # previous + last + 1 start_block = get_previous_block_num(get_last_line(filename)) + 2 upstream_blk = b.get_current_block() checkin_count = 0 print("Our local block: {}".format(start_block)) print("The Upstream block: {}".format(b.get_current_block_num())) print("Progress: {}%\n".format( progress(start_block, b.get_current_block_num()))) with open(filename, 'a+') as file: checkin_ts = time.time() for block in b.stream_from(start_block=start_block, full_blocks=True): if checkin_ts + 10 < time.time(): pdone = progress(int(block['block_id'][:8], base=16), b.get_current_block_num()) dl_rate = checkin_count / 10 print("Rate: {}blk/sec, Progress: {}%".format(dl_rate, pdone)) checkin_ts = time.time() checkin_count = 0 file.write(json.dumps(block, sort_keys=True) + '\n') checkin_count += 1
def get_transactionid(username): time.sleep(6) found = False count = 0 while found == False and count < 5: count = count +1 print ("Attempt to get an ID number: " +str(count)) b = Blockchain() end_block = steem.head_block_number start_block = end_block -100 for block in b.stream_from(start_block=start_block, end_block=end_block, full_blocks=True): transactions = block['transactions'] for transaction in transactions: if (transaction['operations'][0][0] == "custom_json"): if (transaction['operations'][0][1]['required_posting_auths'][0]) == user: print (transaction) if (transaction['operations'][0][1]['required_posting_auths'][0]) == user and (transaction['operations'][0][1]['id'])=="sm_find_match": print (transaction['transaction_id']) trxid = (transaction['transaction_id']) #check if this is a valid id r = requests.get('https://steemmonsters.com/battle/status?id=' +str(trxid)) data = r.json() try: status = data['status'] print ("ID Status:" +str(status)) if int(status) == 1: found = True return (trxid) except: found = False return
def run(): blockchain = Blockchain() head_block = blockchain.get_current_block_num() start_block = head_block - (int(sys.argv[1])) stream = blockchain.stream_from(start_block=start_block) block_count = int(sys.argv[1]) current_block = start_block stats = {} operations = 0 counter = 0 print("Starting from block {} for {} blocks\n".format( start_block, block_count)) for post in stream: if post['block'] != current_block: counter += 1 print("Block {}/{} {:.2f}%".format(counter, block_count, counter / block_count * 100)) current_block = post['block'] elif post['block'] == start_block + block_count: break operation = post['op'][0] if operation not in stats: stats[operation] = 1 else: stats[operation] += 1 operations += 1 print("operations {}\n".format(operations)) for operation in stats: print(operation, stats[operation])
def run(filename): b = Blockchain() # automatically resume from where we left off # previous + last + 1 start_block = get_previous_block_num(get_last_line(filename)) + 2 with open(filename, 'a+') as file: for block in b.stream_from(start_block=start_block, full_blocks=True): file.write(json.dumps(block, sort_keys=True) + '\n')
class Steem_node(): def __init__(self, block, block_count): self.block = block self.end_block = block_count + self.block - 1 self.tag = 'transfer' self.nodes = [ 'https://api.steemit.com', 'https://rpc.buildteam.io', 'https://rpc.steemviz.com' ] self.steem = Steem(nodes=self.nodes) self.b = Blockchain(self.steem) print('Booted\nConnected to: {}'.format(self.nodes[0])) def process_transaction(self, index, block, operation): date = block['timestamp'] to = operation['to'] user = operation['from'] amount = operation['amount'] memo = operation['memo'] db.insert_selection(self.block, index, date, to, user, amount, memo) def run(self): run = 1 # run until end_block while run == 1: try: # stream full blocks starting at start_block stream = self.b.stream_from(start_block=self.block, end_block=self.end_block, full_blocks=True) # process each block indiviudally for block in stream: print('\nBlock: ', self.block) # keep track of transaction index inside block index = 0 # go over each transaction indivually, process if tag is # met and update index for transaction in block['transactions']: if transaction['operations'][0][0] == self.tag: self.process_transaction( index, block, transaction['operations'][0][1]) index += 1 # Check if current block equals the end_block, break if so # else update the current block if self.block == self.end_block: run = 0 else: self.block += 1 except Exception as e: print('Error:', e) continue
class Steem_node(): def __init__(self, block, block_count, tag): self.block = block self.end_block = block_count + self.block - 1 self.tag = tag self.nodes = [ 'https://rpc.buildteam.io', 'https://api.steemit.com', 'https://rpc.steemviz.com' ] self.steem = Steem(nodes=self.nodes) self.b = Blockchain(self.steem) print('Booted\nConnected to: {}'.format(self.nodes[0])) def process_operation(self, operation, tag, timestamp): if tag == 'transfer': transfer = operations.Transfer(operation, self.block, timestamp) transfer.print_operation() elif tag == 'transfer_to_vesting': transfer_to_vesting = operations.Transfer_to_vesting( operation, self.block, timestamp) transfer_to_vesting.print_operation() elif tag == 'withdraw_vesting': withdraw_vesting = operations.Withdraw_vesting( operation, self.block, timestamp) withdraw_vesting.print_operation() elif tag == 'convert': convert = operations.Convert(operation, self.block, timestamp) convert.print_operation() def run(self): run = 1 while run == 1: try: stream = self.b.stream_from(start_block=self.block, end_block=self.end_block, full_blocks=True) for block in stream: print('Block:', self.block) transaction_index = 0 for transaction in block['transactions']: if transaction['operations'][0][0] == self.tag: self.process_operation( transaction['operations'][0][1], self.tag, block['timestamp']) transaction_index += 1 if self.block == self.end_block: run = 0 else: self.block += 1 except Exception as e: print(repr(e)) continue
async def ops_iter(blockchain:Blockchain=None, start_block:int=None): ops_func = blockchain.stream_from( start_block=start_block, batch_operations=False) loop_elapsed = 0 while True: logger.debug('new op', interval=loop_elapsed) loop_start = time.perf_counter() ops = await execute_sync(next, ops_func) yield ops loop_elapsed = time.perf_counter() - loop_start
def listen_steemd(): b = Blockchain() h = b.stream_from( start_block=db_last_block() + 1, full_blocks=True, ) for block in h: num = int(block['previous'][:8], base=16) + 1 print("[LIVE] Got block {} at {} with {} txs".format( num, block['timestamp'], len(block['transactions']))) process_blocks([block])
class Steem_node(): def __init__(self, block, account): self.block = block self.account = account self.nodes = [ 'https://rpc.steemviz.com', 'https://rpc.buildteam.io', 'https://api.steemit.com' ] self.steemPostingKey = os.environ.get('steemPostingKey') self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes) self.b = Blockchain(self.steem) self.queue = classes.Queue(self.steem, self.account) print('\nConnected to: {}'.format(self.nodes[0])) def process_timestamp(self, block): timestamp = block['timestamp'] datetime_object = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S') if datetime_object.second == 54: account = Account(self.account) print(f'\nVoting power: {account.voting_power()}%\n') if account.voting_power() == 100: self.queue.run_voting_round() def process_transaction(self, transaction): if transaction['operations'][0][0] == 'transfer': operation = transaction['operations'][0][1] self.process_transfer(operation) def process_transfer(self, operation): user = operation['from'] amount = operation['amount'] memo = operation['memo'] bid = classes.Bid(user, amount, memo) if operation['to'] == self.account: self.queue.add_bid(bid) def run(self): while True: try: stream = self.b.stream_from(start_block=self.block, full_blocks=True) for block in stream: print('Block:', self.block) self.process_timestamp(block) for transaction in block['transactions']: self.process_transaction(transaction) self.block += 1 except Exception as e: print(repr(e)) continue
def scrape_blockchain(mongo): s = Steem() # see how far behind we are missing = list(range(last_block_num(mongo), s.last_irreversible_block_num)) # if we are far behind blockchain head # split work in chunks of 100 if len(missing) > 100: for batch in partition_all(100, missing): results = s.get_blocks(batch) insert_blocks(mongo, results) # otherwise continue as normal blockchain = Blockchain(mode="irreversible") hist = blockchain.stream_from(start_block=last_block_num(mongo), full_blocks=True) insert_blocks(mongo, hist)
def run(): conn = dbconn.get_connection() lastId = dbutil.get_last_blockId(conn) b = Blockchain() for block in b.stream_from(start_block=lastId + 1, full_blocks=True): print(block['block_id']) btxs = list(map(lambda x: x['operations'][0], block['transactions'])) for tx in btxs: if tx[0] == 'comment' and tx[1]['parent_author'] == '': try: meta = json.loads(tx[1]['json_metadata']) if tx[1]['body'].startswith("@@ ", ) == False: article = Article() article.author = tx[1]['author'] article.subject = tx[1]['title'] article.content = tx[1]['body'] article.tag = meta['tags'] html = mistune.markdown(article.content) article.text = BeautifulSoup( html, "html.parser").get_text().strip() article.preview = article.text[0:100] article.hashId = base64.b64encode( md5(article.preview.encode( "UTF-8")).digest())[0:32] article.created = datetime.datetime.strptime( block['timestamp'], '%Y-%m-%dT%H:%M:%S') article.url = "https://steemkr.com/@" + article.author + "/" + tx[ 1]['permlink'] if "image" in meta and len(meta['image']) > 0: article.img = meta['image'][0] try: if detect(article.text) == "ko": print(meta['tags']) dbutil.insert_article(conn, article) except: pass except: pass dbutil.insert_blockId(conn, block_num_from_hash(block['block_id']), block['timestamp'])
class Steem_node(): def __init__(self, block, block_count, operation): self.block = block self.end_block = block_count + self.block - 1 self.operation = operation self.nodes = ['https://rpc.buildteam.io', 'https://api.steemit.com', 'https://rpc.steemviz.com'] self.steem = Steem(nodes=self.nodes) self.b = Blockchain(self.steem) print('Booted\nConnected to: {}'.format(self.nodes[0])) def run(self): run = 1 while run == 1: try: stream = self.b.stream_from(start_block=self.block, end_block=self.end_block, full_blocks=True) for block in stream: print('\nBlock: ', self.block) transaction_index = 0 for transaction in block['transactions']: if transaction['operations'][0][0] == self.operation: print(transaction_index, transaction['operations'][0][1]) transaction_index += 1 if self.block == self.end_block: run = 0 else: self.block += 1 except Exception as e: continue
class Upvotebot(): def __init__(self): self.node_index = 0 self.nodes = [ 'https://api.steemit.com', 'https://rpc.buildteam.io', 'https://rpc.steemviz.com' ] self.steemPostingKey = os.environ.get('steemPostingKey') self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes) self.b = Blockchain(self.steem, mode='head') self.block = self.b.get_current_block_num() self.upvote_list = [] self.trail_list = [] self.day = None self.hour = None self.timestamp = None self.db = database.Database() # Voting is done via Steemconnect with the steemconnect plugin from # @emre: https://github.com/emre/steemconnect-python-client # Set client_id and client_sercret # Sign up at: https://v2.steemconnect.com/dashboard # # To perform a vote the access_token is fetched from the DB and checked # to not be expired, if so it is renewed and stored inside the DB. # Votes are then removed from the queue and their log is updated. # When a vote fails it gets added to the error database for manual review. def vote(self, voter, author, permlink, weight, type): result = self.db.get_user_auth(voter) access_token, refresh_token, expire_on = result[0] dt = datetime.now() c = Client( client_id=CLIENT_ID, client_secret=CLIENT_SECRET, access_token=access_token, ) try: # Verify access_token if dt > expire_on: access_token = self.refresh_token(refresh_token) result = c.refresh_access_token( refresh_token, "login,vote" # scopes ) access_token = result['access_token'] refresh_token = result['refresh_token'] expires_in = result['expires_in'] self.db.update_authentication_tokens( voter, access_token, refresh_token, expires_in, self.timestamp, ) print('Updated access token\n') # Perform vote if not self.is_already_voted(voter, author, permlink): vote = Vote(voter, author, permlink, weight) result = c.broadcast([vote.to_operation_structure()]) # Log vote if 'error' in result: message = result['error_description'] self.db.add_to_error_log( voter, author, permlink, weight, type, message, self.timestamp, ) else: message = 'Succes' if type == 'vote': self.db.update_log( voter, permlink, message, 'vote_log', ) elif type == 'trail': self.db.update_log( voter, permlink, message, 'trail_log', ) print(f"Voter: {voter}\nAuthor: {author}\n" + f"Permlink: {permlink}\nWeight: {weight}\n" + "Upvote succes\n") except Exception as error: print(voter, author, permlink, weight, error) self.db.add_to_error_log( voter, author, permlink, weight, type, error, self.timestamp, ) # Trails with a delay of 0 are immediatly voted upon. Else the expire_on # time of the vote is calculated and added to the queue. The weight of # vote is relative between the weight of the vote casted by the voter that # is being trailed and the trailer who has set a custom relative weight. def add_trail_to_queue(self, trail, author, permlink, weight): for voter in self.db.get_trail_voters(trail): voter, voter_weight, limit, delay = voter vote_log = self.db.get_trail_log(voter, trail, self.timestamp) upvote_weight = weight / 100 * voter_weight / 100 print(f"\nTrail: {trail}\nVoter: {voter}\nAuthor: {author}\n" + f"Weight: {upvote_weight}\nLimit: {limit}\nDelay: {delay}\n") if delay != 0: self.db.add_trail_to_queue( trail, author, voter, upvote_weight, limit, delay, permlink, self.timestamp, ) elif len(vote_log) < limit: self.db.add_to_trail_log( author, voter, permlink, upvote_weight, self.timestamp, ) self.vote(voter, author, permlink, weight, type) # The time for when a vote is to be made is calculated from the current # block time + the delay. If the delay is 0 an immediate vote is performed. # Votes get added to the queue and log def add_to_queue(self, author, permlink): for vote in self.db.get_votes(author): voter, weight, limit, delay = vote vote_log = self.db.get_vote_log(voter, author, self.timestamp) print(f"\nVoter: {voter}\nWeight: {weight}\nLimit: " + f"{limit}\nDelay: {delay}") if delay != 0: self.db.add_to_queue( author, voter, weight, limit, delay, permlink, self.timestamp, ) elif len(vote_log) < limit: self.db.add_to_vote_log( author, voter, permlink, weight, self.timestamp, ) self.vote(voter, author, permlink, weight, type) # Go thourgh all the active votes and look if the voter has already # voted on this post. def is_already_voted(self, voter, author, permlink): try: identifier = f"@{author}/{permlink}" post = Post(identifier, self.steem) active_votes = post['active_votes'] for vote in active_votes: if vote['voter'] == voter: return 1 return 0 except Exception as error: return 0 # Posts are verified to be the initial post and not an update def is_new_post(self, author, permlink): identifier = f"@{author}/{permlink}" post = Post(identifier, self.steem) created = post['created'] last_update = post['last_update'] if created != last_update: return 0 else: return 1 # Comment is checked to be of the post type and not a comment to a post. # Posts are then added to queue def process_comment(self, comment): parent_author = comment['parent_author'] author = comment['author'] if parent_author == '' and author in self.upvote_list: permlink = comment['permlink'] if self.is_new_post(author, permlink): print(f'\n\n{self.timestamp} Block: {self.block}') print(f"New post\nAuthor: {author}") self.add_to_queue(author, permlink) # Votes are checked for their voter and if the voter is being trailed def process_vote(self, vote): trail = vote['voter'] if trail in self.trail_list: author = vote['author'] permlink = vote['permlink'] weight = vote['weight'] print(f"\n\nTrail: {trail}\nAuthor: {author}\nPermlink: " + f"{permlink}\nWeight: {weight}\n") self.add_trail_to_queue(trail, author, permlink, weight) # Each block the queue is fetched and checked for votes that have reached # their maturity time. def process_queue(self): dt = datetime.strptime(self.timestamp, '%Y-%m-%dT%H:%M:%S') result = self.db.get_queue() if len(result) > 0: for vote in result: id, voter, author, permlink, weight, type, expire_on = vote if dt > expire_on: self.vote(voter, author, permlink, weight, type) self.db.remove_from_queue(id) # Stream block indefinitely, process each block for comments # Stream is set to head mode def run(self): stream = self.b.stream_from( start_block=self.block, full_blocks=True, ) while True: try: for block in stream: # Set block time and process queue self.timestamp = block['timestamp'] print(f'{self.timestamp} Block: {self.block}', end='\r') self.process_queue() # Fetch author to be upvoted list self.upvote_list = [] for author in self.db.get_authors(): author = re.sub( '[^A-Za-z0-9^-]+', '', author[0].lower(), ) self.upvote_list.append(author) # Fetch trails to be tracked self.trail_list = [] for trail in self.db.get_trails(): trail = re.sub( '[^A-Za-z0-9^-]+', '', trail[0].lower(), ) self.trail_list.append(trail) # Look at each single transaxtion for transaction in block['transactions']: transaction_type = transaction['operations'][0][0] if transaction_type == 'comment': comment = transaction['operations'][0][1] self.process_comment(comment) elif transaction_type == 'vote': vote = transaction['operations'][0][1] self.process_vote(vote) self.block += 1 except Exception as e: print(e) # common exception for api.steemit.com # when the head block is reached s = ("TypeError('an integer is" " required (got type NoneType)',)") if repr(e) == s: time.sleep(1) else: stream = self.b.stream_from( start_block=self.block, full_blocks=True, ) continue
class Steem_node(): def __init__(self, acount, number): self.tag = 'vote' self.account = account self.number = number self.nodes = [ 'https://api.steemit.com', 'https://rpc.buildteam.io', 'https://rpc.steemviz.com' ] self.steem = Steem(nodes=self.nodes) self.b = Blockchain(self.steem) self.timestamp = None self.block = self.b.get_current_block_num() print('Booted\nConnected to: {}'.format(self.nodes[0])) def send_sms(self, memo): print(twilio_api.send_sms(self.number, memo)) # Convert rshares to dollar value # Author: emrebeyler # Source: https://goo.gl/dV7yVa def convert_rshares(self, rshares): reward_fund = self.steem.get_reward_fund() reward_balance, recent_claims = (reward_fund["reward_balance"], reward_fund["recent_claims"]) base_price = self.steem.get_current_median_history_price()["base"] fund_per_share = Amount(reward_balance).amount / float(recent_claims) payout = float(rshares) * fund_per_share * Amount(base_price).amount return payout def process_vote(self, vote): print(vote) voter = vote['voter'] author = vote['author'] permlink = vote['permlink'] weight = vote['weight'] # Verify that utopian-io is the voter and that is votes on account if voter == 'utopian-io' and author == account: print(f"\nVoter: {voter}\nAuthor: {author}\nPermlink: {permlink}" + f"\nWeight: {weight}\n{self.timestamp}\n") # Construct post from the permlink to retrieve vote rshares post = Post(f'@{self.account}/{permlink}', self.steem) # Look for utopian in active_votes active_votes = post['active_votes'] for vote in active_votes: if vote['voter'] == 'utopian-io': payout = self.convert_rshares(vote['rshares']) # Construct sms url = f"http://steemit.com/@{self.account}/{permlink}" body = (f"Upvote from Utopian-io for ${payout:.2f}\n" + f"\n{url}\n\nBlock: {self.block}" + f"\n{self.timestamp}") self.send_sms(body) def run(self): print(self.block) while True: try: # stream full blocks stream = self.b.stream_from(start_block=self.block, full_blocks=True) # process each block indiviudally for block in stream: self.timestamp = block['timestamp'] print(f'{self.timestamp} Block: {self.block}', end='\r') # go over each transaction indivually, process if tag is # met and update index for transaction in block['transactions']: if transaction['operations'][0][0] == self.tag: transfer = transaction['operations'][0][1] self.process_vote(transfer) self.block += 1 except Exception as e: # common exception for api.steemit.com # when the head block is reached s = ("TypeError('an integer is" " required (got type NoneType)',)") if repr(e) == s: time.sleep(3)
class FunfunPosting(object): def __init__(self): self.steem = Steem() self.blockchain = Blockchain() self.account = config.ACCOUNT['user'] connObj = DBActions() self.conn = connObj.getConnection() def _getLastBlockId(self): cursor = self.conn.cursor() query = """ SELECT max(last_block) as latest FROM last_block """ cursor.execute(query) ret = cursor.fetchone() if ret is None or len(ret) < 1: return 1 return ret['latest'] def _updateLastBlockId(self, block_no): cursor = self.conn.cursor() query = """ UPDATE last_block SET last_block = %s WHERE id = 1 """ try: cursor.execute(query, (block_no, )) except: traceback.print_exc() self.conn.rollback() return False return True def _getUserId(self, user_text_id): cursor = self.conn.cursor() query = """ SELECT id FROM users WHERE steemid = %s LIMIT 1 """ cursor.execute(query, (user_text_id, )) ret = cursor.fetchone() if ret is None or len(ret) < 1: return -1 return ret['id'] def findUser(self, user_text_id): cursor = self.conn.cursor() query = """ SELECT count(*) as cnt FROM users WHERE steemid = %s """ cursor.execute(query, (user_text_id, )) ret = cursor.fetchone() if ret is None or len(ret) < 1: return False return True if ret['cnt'] == 1 else False def addUser(self, user_text_id): cursor = self.conn.cursor() query = """ INSERT INTO users (steemid, added_at) VALUES (%s, CURRENT_TIMESTAMP) """ try: cursor.execute(query, (user_text_id, )) except: traceback.print_exc() self.conn.rollback() return False self.conn.commit() return True def deleteUser(self, user_text_id): cursor = self.conn.cursor() query = """ DELETE FROM users WHERE steemid = %s """ try: cursor.execute(query, (user_text_id, )) except: traceback.print_exc() self.conn.rollback() return False self.conn.commit() return True def getUserLists(self): cursor = self.conn.cursor() query = """ SELECT steemid FROM users """ cursor.execute(query) ret = cursor.fetchall() if ret is None or len(ret) < 1: return [] result = ['@{}'.format(item['steemid']) for item in ret] return result def getThemeLists(self): cursor = self.conn.cursor() query = """ SELECT * FROM theme """ cursor.execute(query) ret = cursor.fetchall() if ret is None or len(ret) < 1: return [] result = [{ "id": item['id'], "theme_name": item['theme_name'], "added_time": item['added_time'] } for item in ret] return result def addTheme(self, theme_name): cursor = self.conn.cursor() query = """ INSERT INTO theme (theme_name, added_time) VALUES (%s, CURRENT_TIMESTAMP) """ try: cursor.execute(query, (theme_name, )) except: traceback.print_exc() self.conn.rollback() return False self.conn.commit() return True def modifyTheme(self, theme_id, theme_name): cursor = self.conn.cursor() query = """ UPDATE theme SET theme_name = %s WHERE id = %s """ try: cursor.execute(query, ( theme_name, theme_id, )) except: traceback.print_exc() self.conn.rollback() return False self.conn.commit() return True def deleteTheme(self, theme_name): cursor = self.conn.cursor() query = """ DELETE FROM theme WHERE theme_name = %s """ try: cursor.execute(query, (theme_name, )) except: traceback.print_exc() self.conn.rollback() return False self.conn.commit() return True def _findThemeId(self, theme_name): cursor = self.conn.cursor() query = """ SELECT id FROM theme WHERE theme_name = %s LIMIT 1 """ cursor.execute(query, (theme_name, )) ret = cursor.fetchone() if ret is None or len(ret) < 1: return -1 return ret['id'] def _insertPostInfo(self, user_id, theme_id, user_text_id, permlink, tags, written_at, is_regular=False): cursor = self.conn.cursor() query = """ INSERT INTO posts (user_id, theme_id, user_text_id, permlink, tags, written_at, is_voted, is_regular) VALUES (%s, %s, %s, %s, %s, %s, 0, %s) """ try: cursor.execute(query, ( user_id, theme_id, user_text_id, permlink, tags, written_at, is_regular, )) except pymysql.err.IntegrityError: print("Duplicated or revised post") return False, 1 except: traceback.print_exc() self.conn.rollback() return False, 2 self.conn.commit() return True, 0 def _voting(self, user_text_id, permlink, is_regular=False, is_voted=False): if is_voted == True: return True cursor = self.conn.cursor() query = """ UPDATE posts SET is_voted = 1 WHERE user_text_id = %s AND permlink = %s """ try: cursor.execute(query, ( user_text_id, permlink, )) except: traceback.print_exc() self.conn.rollback() return False ## VOTE post = "@{}/{}".format(user_text_id, permlink) try: """ { "voter": "badfatcat", "author": "claudene", "permlink": "update-the-new-coins-in-the-philippines-the-p10-pesos-and-p5-pesos-2018-20171231t62751604z", "weight": 10000, "_id": "1af544c59ea75a816ed756578105d9befa442224", "type": "vote", "timestamp": "2018-01-15 14:32:33", "block_num": 19001685, "trx_id": "1d0490ac6aea885b373a5dc0f1bebb22d9f9fd96" } """ rate = 50 if is_regular == True: rate = 100 ret = self.steem.vote(post, rate, "funfund") except steembase.exceptions.RPCError: print("Already voted!") except: traceback.print_exc() self.conn.rollback() return False query = """ UPDATE posts SET is_voted = 2, voted_at = CURRENT_TIMESTAMP WHERE user_text_id = %s AND permlink = %s """ try: cursor.execute(query, ( user_text_id, permlink, )) except: traceback.print_exc() self.conn.rollback() return False self.conn.commit() time.sleep(21) return True def filterPosts(self, post_arr): """ { 'signatures':[ '1f2e21012c8ce7f64275b2144bddbd114c8d918dd9ba031274496bf7144c968af1443f2a99efe47ad8b609357ef157e14cc692a5e8b8d52da83d84aa5f3d25b795' ], 'expiration':'2018-06-27T16:50:33', 'transaction_num':24, 'extensions':[ ], 'operations':[ [ 'comment', { 'body':'[![Thumbnail](https://images.dlive.io/cf1db880-7a28-11e8-9118-0242ac110002)](https://dlive.io/livestream/gregoryarts/cf6aaf79-7a28-11e8-9907-0242ac110003)\n\nMinecraft 1.12.2\n\nnotifications thanks :) https://zygmunt.pro/apps/dcr/ \n\nhttps://tipanddonation.com/gregoryarts \n\nMój sprzęt \n\nProcesor AMD FX-8320 8 x 4,5 Ghz \nPłyta główna MSI 970 Gaming (MS-7693) \nPamięć ram HyperX Savage 8GB 2400MHz DDR3 CL11 (2x4GB) \nGrafika Gigabyte GeForce GTX 1050 Ti OC 4GB GDDR5 \nZasilacz Radix ECO III 650W Obudowa z okienkiem ;) (projekt własny - filmy z jej tworzeniem dostępne na kanale d-tube)\n\nMy live stream is at [DLive](https://dlive.io/livestream/gregoryarts/cf6aaf79-7a28-11e8-9907-0242ac110003)', 'json_metadata':'{"tags":["dlive","dlive-broadcast","game","minecraft","gamplay","mods","polish"],"app":"dlive/0.1","format":"markdown","language":"en","thumbnail":"https://images.dlive.io/cf1db880-7a28-11e8-9118-0242ac110002"}', 'title':'[PL] Minecraft - granie na modach #001', 'author':'gregoryarts', 'parent_author':'', 'permlink':'cf6aaf79-7a28-11e8-9907-0242ac110003', 'parent_permlink':'dlive' } ] ], 'ref_block_num':35368, 'transaction_id':'b02bd95c0a07850ceb606771e09ea733679dbf25', 'block_num':23693882, 'ref_block_prefix':4217244095 } """ filtered_result = [] for item in post_arr: try: operations = item.get('operations') if operations == None: print("An important info is null") continue content_obj = operations[0][1] parent_author = content_obj.get('parent_author') try: json_metadata = json.loads(content_obj['json_metadata']) tags = json_metadata.get('tags', []) except: continue author = content_obj.get('author') original_theme = content_obj.get('title') permlink = content_obj.get('permlink') written_at = item.get('expiration') if author == None or original_theme == None or permlink == None or parent_author == None: print("@{}/{}: An important info is null".format( author, permlink)) continue except: traceback.print_exc() print(content_obj) print("Error in TX!!") sys.exit(1) # Tag check if "kr-funfun" not in tags: print("@{}/{}: No 'kr-funfun' tag in the post".format( author, permlink)) continue # User check user_id = self._getUserId(author) if user_id == -1: print("@{}/{}: User {} is not member of kr-funfun".format( author, permlink, author)) continue # Theme check is_regular = False extract_theme_obj = re.search(r':(.*?)\]', original_theme) theme_id = -1 if extract_theme_obj != None: theme = extract_theme_obj.group(1).strip() theme_id = self._findThemeId(theme) if theme_id == -1: is_regular = False else: is_regular = True string_tags = ','.join(tags) is_ok, code = self._insertPostInfo(user_id, theme_id, author, permlink, string_tags, written_at, is_regular) is_voted = False if is_ok == False and code >= 2: self.conn.rollback() print("Something wrong in writing funfun posts") sys.exit(1) elif is_ok == False and code == 1: self.conn.rollback() is_voted = True else: is_voted = False filtered_result.append({ "user_text_id": author, "permlink": permlink, "is_regular": is_regular, "is_voted": is_voted }) self.conn.commit() return filtered_result def main(self): last_block_id = self._getLastBlockId() # Block streaming.. for block in self.blockchain.stream_from(start_block=last_block_id, full_blocks=True): for tx in block['transactions']: ret = [] tx_type = tx['operations'][0][0] title = tx['operations'][0][1].get('title') if tx_type == 'comment' and title != '' and title != None: ret.append(tx) last_block_id = tx['block_num'] filtered_posts = self.filterPosts(ret) if len(filtered_posts) > 0: print("Gathered {} kr-funfun posts!".format( len(filtered_posts))) for item in filtered_posts: print("Voting @{}/{}..".format(item['user_text_id'], item['permlink'])) is_ok = self._voting(item['user_text_id'], item['permlink'], item['is_regular'], item['is_voted']) if is_ok == False: print("Something wrong in @{}/{}".format( item['user_text_id'], item['permlink'])) self.conn.rollback() sys.exit(1) self.conn.commit() self._updateLastBlockId(last_block_id) self.conn.commit()
'interval', seconds=15, id='process_vote_queue') scheduler.add_job(process_rewards_pools, 'interval', minutes=10, id='process_rewards_pools') scheduler.start() quick = False # last_save_height_time = time.time() last_save_block_processed = last_block_processed while True: for block in b.stream_from(start_block=last_block_processed, batch_operations=False, full_blocks=True): if (len(block) > 0): block_num = block_num_from_hash(block['block_id']) timestamp = block['timestamp'] # If behind by more than X (for initial indexes), set quick to true to prevent unneeded past operations remaining_blocks = props[ 'last_irreversible_block_num'] - block_num if remaining_blocks > quick_value: quick = True dt = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S') l('----------------------------------') l('#{} - {} - {} ops ({} remaining|quick:{})'.format( block_num, dt, len(block['transactions']), remaining_blocks, quick)) for idx, tx in enumerate(block['transactions']):
class YoBlockchainFollower(YoBaseService): service_name = 'blockchain_follower' def __init__(self, **kwargs): super().__init__(**kwargs) # init blockchain steemd_url = self.yo_app.config.steemd_url self.steemd = steem.steemd.Steemd(nodes=[steemd_url]) self.blockchain = Blockchain(steemd_instance=self.steemd) self.start_block = self.yo_app.config.steemd_start_block self.last_block_num_handled = 0 # init ops handlers self.op_map = defaultdict(list) self.op_map.update({ 'vote': [self.handle_vote], 'account_update': [self.handle_account_update], 'transfer': [self.handle_send, self.handle_receive], 'custom_json': [self.handle_follow, self.handle_resteem], 'withdraw_vesting': [self.handle_power_down], 'comment': [self.handle_mention, self.handle_comment] }) start_block = self.get_start_block() self.ops_func = self.blockchain.stream_from( start_block=start_block, batch_operations=True) self.ops = lambda: self.execute_sync(next, self.ops_func) def get_start_block(self): start_block = None try: if isinstance(self.start_block, int) and self.start_block < 0: start_block = self.blockchain.get_current_block_num() - self.start_block except Exception: self.log.exception('service error') start_block = None self.log.debug('get_start_block', start_block=start_block) return start_block async def store_notification(self, notifications): return await self.db.create_notifications(notifications) # pylint gets confused about these for some reason # pylint: disable=no-member async def handle_vote(self, op): vote_info = op['op'][1] self.log.info( 'handle_vote', permlink=vote_info['permlink'], author=vote_info['author'], voter=vote_info['voter'], weight=vote_info['weight']) return [ dict( trx_id=op['trx_id'], from_username=vote_info['voter'], to_username=vote_info['author'], json_data=json.dumps(vote_info), notify_type=VOTE, priority_level=Priority.LOW.value) ] async def handle_follow(self, op): if op['op'][1]['id'] != 'follow': self.log.debug('handle_follow noop') return [] op_data = op['op'][1] follow_data = json.loads(op_data['json']) if follow_data[0] != 'follow': return [] follower = follow_data[1]['follower'] following = follow_data[1]['following'] if len(op_data['required_posting_auths']) != 1: self.log.error('inavlid follow op, got %d posting auths, expected 1', op_data['required_posting_auths']) return [] if op_data['required_posting_auths'][0] != follower: self.log.error('invalid follow op, follower must be signer') return [] self.log.debug('handle_follow', follower=follower, following=following) return [ dict( trx_id=op['trx_id'], from_username=follower, to_username=following, json_data=json.dumps(follow_data[1]), notify_type=FOLLOW, priority_level=Priority.LOW.value) ] async def handle_account_update(self, op): op_data = op['op'][1] self.log.debug('handle_account_update', account=op_data['account']) return [ dict( trx_id=op['trx_id'], to_username=op_data['account'], json_data=json.dumps(op_data), notify_type=ACCOUNT_UPDATE, priority_level=Priority.LOW.value) ] async def handle_send(self, op): op_data = op['op'][1] send_data = { 'amount': op_data['amount'], 'from': op_data['from'], 'memo': op_data['memo'], 'to': op_data['to'], } self.log.debug( 'handle_send', _from=send_data['from'], amount=send_data['amount'], to=send_data['to']) return [ dict( trx_id=op['trx_id'], to_username=send_data['from'], json_data=json.dumps(send_data), notify_type=SEND_STEEM, priority_level=Priority.LOW.value) ] async def handle_receive(self, op): op_data = op['op'][1] receive_data = { 'amount': op_data['amount'], 'from': op_data['from'], 'memo': op_data['memo'], 'to': op_data['to'], } self.log.debug( 'handle_receive', to=receive_data['to'], amount=receive_data['amount'], _from=receive_data['from']) return [ dict( trx_id=op['trx_id'], to_username=receive_data['to'], from_username=receive_data['from'], json_data=json.dumps(receive_data), notify_type=RECEIVE_STEEM, priority_level=Priority.LOW.value) ] async def handle_power_down(self, op): op_data = op['op'][1] self.log.debug( 'handle_power_down', account=op_data['account'], amount=op_data['vesting_shares']) return [ dict( trx_id=op['trx_id'], to_username=op_data['account'], json_data=json.dumps(op_data), notify_type=POWER_DOWN, priority_level=Priority.LOW.value) ] async def handle_mention(self, op): comment_data = op['op'][1] haystack = comment_data['body'] + '\n' data = { 'author': comment_data['author'], 'permlink': comment_data['permlink'], } notifications = [] for match in set(re.findall(MENTION_PATTERN, haystack)): self.log.debug('handle_mention', author=data['author'], mentioned=match) notifications.append( dict( trx_id=op['trx_id'], to_username=match, from_username=data['author'], json_data=json.dumps(data), notify_type=MENTION, priority_level=Priority.LOW.value)) return notifications async def handle_comment(self, op): self.log.debug('handle_comment', op=['op'][0]) op_data = op['op'][1] if op_data['parent_author'] == '': # top level post return [] parent_id = '@' + op_data['parent_author'] + '/' + op_data['parent_permlink'] parent = steem.post.Post(parent_id) note_type = COMMENT_REPLY if parent.is_comment() else POST_REPLY self.log.debug( 'handle_comment', note_type=note_type, author=op_data['author'], parent_id=parent_id) return [ dict( trx_id=op['trx_id'], to_username=op_data['parent_author'], from_username=op_data['author'], json_data=json.dumps(op_data), notify_type=note_type, priority_level=Priority.LOW.value) ] async def handle_resteem(self, op): op_data = op['op'][1] resteem_data = json.loads(op_data['json']) if resteem_data[0] != 'reblog': self.log.debug('handle_resteem noop') return [] account = resteem_data[1]['account'] author = resteem_data[1]['author'] permlink = resteem_data[1]['permlink'] if len(op_data['required_posting_auths']) != 1: self.log.error('inavlid resteem op, got %d posting auths, expected 1', op_data['required_posting_auths']) return [] if op_data['required_posting_auths'][0] != account: self.log.error('invalid resteem op, account must be signer') return [] self.log.debug( 'handle_resteem', account=account, author=author, permlink=permlink) return [ dict( trx_id=op['trx_id'], from_username=account, to_username=author, json_data=json.dumps(resteem_data[1]), notify_type=RESTEEM, priority_level=Priority.LOW.value) ] # pylint: enable=no-member async def notify(self, blockchain_op): """ Handle notification for a particular op """ op_type = blockchain_op['op'][0] futures = [handler(blockchain_op) for handler in self.op_map[op_type]] if futures: self.log.debug( 'operation triggering handlers', op_type=op_type, handlers=futures) return await asyncio.gather(*futures) else: self.log.debug('skipping operation', op_type=op_type) return [] async def ops_iter(self): start_block = self.get_start_block() ops_func = self.blockchain.stream_from( start_block=start_block, batch_operations=True) while True: ops = await self.execute_sync(next, ops_func) yield ops async def main_task(self): self.log.debug('main task executed') async for ops in self.ops_iter(): block_num = ops[0]['block'] self.log.debug('main task', op_in_block=len(ops), block_num=block_num) unstored_notifications = await asyncio.gather( *[self.notify(op) for op in ops]) combined_notifications = list( itertools.chain(*itertools.chain(*unstored_notifications))) logger.debug( 'main_task', block_num=block_num, op_count=len(ops), unstored_count=sum(map(len, unstored_notifications)), combined_notifications=len(combined_notifications)) resp = await self.store_notification(combined_notifications) if resp: self.last_block_num_handled = block_num
class Upvotebot(): def __init__(self, account): self.block = None self.node_index = 0 self.list = [ 'https://api.steemit.com', 'https://rpc.buildteam.io', 'https://rpc.steemviz.com' ] self.nodes = [self.list[self.node_index]] self.steemPostingKey = os.environ.get('steemPostingKey') self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes) self.b = Blockchain(self.steem) self.account = account self.upvote_list = json.load(open('upvote_list.json')) self.day = None self.hour = None self.queue = classes.Queue(account, self.steem) print('Running photobot \nConnected to: {}'.format(self.nodes[0])) # set block times def set_times(self, datetime_object): self.day = datetime_object.day self.hour = datetime_object.hour print('Block times set') # process block time def time_handler(self, datetime_object): if not self.day and not self.hour: self.set_times(datetime_object) day = datetime_object.day hour = datetime_object.hour # clear memory for a new day if day is not self.day: print("\nNew day: cleared memory\n") self.queue.reset() self.day = day # reload the upvote_list every hour if hour is not self.hour: print("\nNew hour: reloaded upvote list\n") self.upvote_list = json.load(open('upvote_list.json')) self.hour = hour # check if the operation is a post and not a comment, also check # if the author is in the upvote list. def process_operation(self, operation, timestamp): parent_author = operation['operations'][0][1]['parent_author'] if parent_author == '': author = operation['operations'][0][1]['author'] permlink = operation['operations'][0][1]['permlink'] identifier = '@{}/{}'.format(author, permlink) if author in self.upvote_list: self.queue.add_to_queue(identifier, timestamp) # in case one of the nodes fail switch over to the next node def reconfigure_node(self): self.node_index = (self.node_index + 1) % len(self.list) self.nodes = [self.list[self.node_index]] self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes) self.b = Blockchain(self.steem) self.queue.steem = self.steem print('New node: {}\n'.format(self.nodes)) # Stream block indefinitely, process each transaction in each block def run(self, block_num, upvote_age): # filter command line arguments if block_num == 'head': self.block = self.blockchain.get_current_block_num() else: self.block = int(block_num) # set and convert upvote age to seconds self.queue.upvote_age = upvote_age * 60 while True: try: stream = self.b.stream_from(start_block=self.block, full_blocks=True) for block in stream: # process the timestamp timestamp = block['timestamp'] print(self.block, timestamp) self.queue.process_queue(timestamp) datetime_object = datetime.strptime( timestamp, '%Y-%m-%dT%H:%M:%S') self.time_handler(datetime_object) # go over each transaction and process comments for operation in block['transactions']: operation_type = operation['operations'][0][0] if operation_type == 'comment': self.process_operation(operation, timestamp) self.block += 1 except Exception as e: # common exception for api.steemit.com # when the head block is reached s = ("TypeError('an integer is" " required (got type NoneType)',)") if repr(e) == s: time.sleep(3) else: print(self.block, repr(e)) self.reconfigure_node() continue
process_platform_history() process_global_props() process_rewards_pools() rebuild_forums_cache() rebuild_bots_cache() scheduler = BackgroundScheduler() scheduler.add_job(process_global_props, 'interval', seconds=9, id='process_global_props') scheduler.add_job(rebuild_forums_cache, 'interval', minutes=1, id='rebuild_forums_cache') scheduler.add_job(rebuild_bots_cache, 'interval', minutes=1, id='rebuild_bots_cache') scheduler.add_job(process_vote_queue, 'interval', seconds=15, id='process_vote_queue') scheduler.add_job(process_rewards_pools, 'interval', minutes=10, id='process_rewards_pools') scheduler.start() quick = False for block in b.stream_from(start_block=last_block_processed, full_blocks=True): if(len(block['transactions']) > 0): block_num = block_num_from_hash(block['block_id']) timestamp = block['timestamp'] # If behind by more than X (for initial indexes), set quick to true to prevent unneeded past operations remaining_blocks = props['last_irreversible_block_num'] - block_num if remaining_blocks > quick_value: quick = True dt = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S') l('----------------------------------') l('#{} - {} - {} ops ({} remaining|quick:{})'.format(block_num, dt, len(block['transactions']), remaining_blocks, quick)) for idx, tx in enumerate(block['transactions']): txid = block['transaction_ids'][idx] # Is this a group of ops for the forum? if len(tx['operations']) > 1: