def test_stream_threading(self): bts = self.bts b = Blockchain(steem_instance=bts) ops_stream_no_threading = [] opNames = ["transfer", "vote"] block_num_list2 = [] for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, threading=False): ops_stream_no_threading.append(op) if op["block_num"] not in block_num_list2: block_num_list2.append(op["block_num"]) for n in range(5): ops_stream = [] block_num_list = [] for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, threading=True, thread_num=8): ops_stream.append(op) if op["block_num"] not in block_num_list: block_num_list.append(op["block_num"]) self.assertEqual(ops_stream[0]["block_num"], ops_stream_no_threading[0]["block_num"]) self.assertEqual(ops_stream[-1]["block_num"], ops_stream_no_threading[-1]["block_num"]) self.assertEqual(len(ops_stream_no_threading), len(ops_stream)) self.assertEqual(len(block_num_list), len(block_num_list2)) for i in range(len(block_num_list)): self.assertEqual(block_num_list[i], block_num_list2[i])
class Crawler: def __init__(self, name, blockchain): self.__name = name if blockchain == "hive": self._blockchain = Blockchain( blockchain_instance=blockchains.HIVE_INSTANCE) pass elif blockchain == "steem": self._blockchain = Blockchain( blockchain_instance=blockchains.STEEM_INSTANCE) else: raise NotImplementedError def _stream(self, opNames=[]): if hasattr(self, "start") and hasattr(self, "stop"): # get starting block id start_block_id = self._blockchain.get_estimated_block_num( self.start, accurate=True) stop_block_id = self._blockchain.get_estimated_block_num( self.stop, accurate=True) # looping trough generator for op_json in self._blockchain.stream(opNames=opNames, start=start_block_id, stop=stop_block_id): yield op_json else: print( "[DEBUG] Timeframe was not set in criteria, direct streaming used" ) # looping trough generator for op_json in self._blockchain.stream(opNames=opNames): yield op_json def set_timeframe(self, start, stop): if isinstance(start, datetime.datetime): self.start = start else: raise TypeError( "start argument must be an instance of datetime.datetime") if isinstance(stop, datetime.datetime): self.stop = stop else: raise TypeError( "stop argument must be an instance of datetime.datetime") if start >= stop: raise RuntimeError("stop({}) < start({})".format(stop, start))
def read_missing_block_data(self, start_block=None): # Go trough all transfer ops cnt = 0 blockchain = Blockchain() current_block = blockchain.get_current_block_num() if start_block is not None: trx_num = start_block["trx_num"] block_num = start_block["block_num"] start_block = block_num # print("account %s - %d" % (account["name"], start_block)) else: trx_num = 0 block_num = 0 start_block = current_block - self.block_history data = [] for op in blockchain.stream(start=start_block, stop=current_block, only_ops=self.only_ops, only_virtual_ops=self.only_virtual_ops): if op["block_num"] < start_block: # last_block = op["block"] continue elif op["block_num"] == start_block: if op["trx_num"] <= trx_num: continue data.append(op) cnt += 1 return data
def test_stream_batch(self): bts = self.bts b = Blockchain(steem_instance=bts) ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False): ops_stream.append(op) self.assertTrue(ops_stream[0]["block_num"] >= self.start) self.assertTrue(ops_stream[-1]["block_num"] <= self.stop) op_stat = b.ops_statistics(start=self.start, stop=self.stop) self.assertEqual(op_stat["vote"] + op_stat["transfer"], len(ops_stream)) ops_blocks = [] for op in b.blocks(start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False): ops_blocks.append(op) op_stat4 = {"transfer": 0, "vote": 0} self.assertTrue(len(ops_blocks) > 0) for block in ops_blocks: for tran in block["transactions"]: for op in tran['operations']: if isinstance(op, dict) and "type" in op and "value" in op: op_type = op["type"] if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation": op_type = op_type[:-10] if op_type in opNames: op_stat4[op_type] += 1 elif op[0] in opNames: op_stat4[op[0]] += 1 self.assertTrue(block.identifier >= self.start) self.assertTrue(block.identifier <= self.stop) self.assertEqual(op_stat["transfer"], op_stat4["transfer"]) self.assertEqual(op_stat["vote"], op_stat4["vote"])
def profiling(name_list): stm = Steem() set_shared_steem_instance(stm) del stm print("start") for name in name_list: print("account: %s" % (name)) acc = Account(name) max_index = acc.virtual_op_count() print(max_index) stopTime = datetime(2018, 4, 22, 0, 0, 0) hist_elem = None for h in acc.history_reverse(stop=stopTime): hist_elem = h print(hist_elem) print("blockchain") blockchain_object = Blockchain() current_num = blockchain_object.get_current_block_num() startBlockNumber = current_num - 20 endBlockNumber = current_num block_elem = None for o in blockchain_object.stream(start=startBlockNumber, stop=endBlockNumber): print("block %d" % (o["block_num"])) block_elem = o print(block_elem)
def test_stream_threading(self): bts = self.bts b = Blockchain(steem_instance=bts) ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=self.start, stop=self.stop): ops_stream.append(op) self.assertTrue(len(ops_stream) > 0) op_stat = b.ops_statistics(start=self.start, stop=self.stop) ops_blocks = [] for op in b.blocks(start=self.start, stop=self.stop, threading=True, thread_num=8): ops_blocks.append(op) op_stat4 = {"transfer": 0, "vote": 0} self.assertTrue(len(ops_blocks) > 0) for block in ops_blocks: for tran in block["transactions"]: for op in tran['operations']: if op[0] in opNames: op_stat4[op[0]] += 1 self.assertTrue(block.identifier >= self.start) self.assertTrue(block.identifier <= self.stop) self.assertEqual(op_stat["transfer"], op_stat4["transfer"]) self.assertEqual(op_stat["vote"], op_stat4["vote"])
def test_stream2(self): bts = self.bts b = Blockchain(steem_instance=bts) stop_block = b.get_current_block_num() start_block = stop_block - 10 ops_stream = [] for op in b.stream(start=start_block, stop=stop_block): ops_stream.append(op) self.assertTrue(len(ops_stream) > 0)
def listen_blockchain_ops(opNames: list): """Listens to Steem blockchain and yields specified operations. :param opNames: List of operations to yield :type opNames: list """ bc = Blockchain(mode="head") block_num = bc.get_current_block_num() for op in bc.stream(opNames=opNames, start=block_num, threading=True): yield op
def stream_blocks(self, start=None): blockchain = Blockchain() current_block = blockchain.get_current_block_num() ops = [] for op in blockchain.stream(start=start, stop=current_block, only_ops=self.only_ops, only_virtual_ops=self.only_virtual_ops): ops.append(op) return ops
def test_stream(self, node_param): if node_param == "non_appbase": bts = self.bts else: bts = self.appbase b = Blockchain(steem_instance=bts) ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=self.start, stop=self.stop): ops_stream.append(op) self.assertTrue(len(ops_stream) > 0) op_stat = b.ops_statistics(start=self.start, stop=self.stop) op_stat2 = {"transfer": 0, "vote": 0} for op in ops_stream: self.assertIn(op["type"], opNames) op_stat2[op["type"]] += 1 self.assertTrue(op["block_num"] >= self.start) self.assertTrue(op["block_num"] <= self.stop) self.assertEqual(op_stat["transfer"], op_stat2["transfer"]) self.assertEqual(op_stat["vote"], op_stat2["vote"]) ops_ops = [] for op in b.ops(start=self.start, stop=self.stop): ops_ops.append(op) self.assertTrue(len(ops_ops) > 0) op_stat3 = {"transfer": 0, "vote": 0} for op in ops_ops: if op["op"][0] in opNames: op_stat3[op["op"][0]] += 1 self.assertTrue(op["block_num"] >= self.start) self.assertTrue(op["block_num"] <= self.stop) self.assertEqual(op_stat["transfer"], op_stat3["transfer"]) self.assertEqual(op_stat["vote"], op_stat3["vote"]) ops_blocks = [] for op in b.blocks(start=self.start, stop=self.stop): ops_blocks.append(op) op_stat4 = {"transfer": 0, "vote": 0} self.assertTrue(len(ops_blocks) > 0) for block in ops_blocks: for tran in block["transactions"]: for op in tran['operations']: if op[0] in opNames: op_stat4[op[0]] += 1 self.assertTrue(block.identifier >= self.start) self.assertTrue(block.identifier <= self.stop) self.assertEqual(op_stat["transfer"], op_stat4["transfer"]) self.assertEqual(op_stat["vote"], op_stat4["vote"]) ops_blocks = [] for op in b.blocks(): ops_blocks.append(op) break self.assertTrue(len(ops_blocks) == 1)
class SteemStream: def __init__(self, operations=[]): self.blockchain = Blockchain(mode="head") self.operations = operations self.last_streamed_block = 0 self.max_batch_size = 50 self.threading = False def run(self, callback=default_callback, lookback=0, start=-1, days=-1): try: if lookback > 0 or start > 0 or days > 0: if self.last_streamed_block == 0: if start > 0: start_block = start elif lookback > 0: start_block = self.blockchain.get_current_block_num() - int(lookback) #200000 else: start_date = days_ago(days) start_block = self.blockchain.get_estimated_block_num(start_date) else: start_block = self.last_streamed_block + 1 stop_block = self.blockchain.get_current_block_num() logger.info("Streaming for operations {} has started from block {} to {}".format(self.operations, start_block, stop_block)) for ops in self.blockchain.stream(opNames=self.operations, start=start_block, stop=stop_block, max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8): try: callback(ops) except: logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc())) else: logger.info("Streaming for operations {} has started from the latest blocks".format(self.operations)) for ops in self.blockchain.stream(opNames=self.operations, max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8): try: callback(ops) except: logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc())) except: logger.error("Failed when streaming operations {} with error: {}".format(self.operations, traceback.format_exc()))
def test_stream2(self, node_param): if node_param == "normal": bts = self.bts else: bts = self.testnet b = Blockchain(steem_instance=bts) stop_block = b.get_current_block_num() start_block = stop_block - 10 ops_stream = [] for op in b.stream(start=start_block, stop=stop_block): ops_stream.append(op) self.assertTrue(len(ops_stream) > 0)
def stream_votes(stm, threading, thread_num): b = Blockchain(steem_instance=stm) opcount = 0 start_time = time.time() for op in b.stream(start=23483000, stop=23483200, threading=threading, thread_num=thread_num, opNames=['vote']): sys.stdout.write("\r%s" % op['block_num']) opcount += 1 now = time.time() total_duration = now - start_time print(" votes: %d, time %.2f" % (opcount, total_duration)) return opcount, total_duration
def test_stream_batch2(self): bts = self.bts b = Blockchain(steem_instance=bts) ops_stream = [] start_block = 25097000 stop_block = 25097100 opNames = ["account_create", "custom_json"] for op in b.stream(start=int(start_block), stop=int(stop_block), opNames=opNames, max_batch_size=50, threading=False, thread_num=8): ops_stream.append(op) self.assertTrue(ops_stream[0]["block_num"] >= start_block) self.assertTrue(ops_stream[-1]["block_num"] <= stop_block) op_stat = b.ops_statistics(start=start_block, stop=stop_block) self.assertEqual(op_stat["account_create"] + op_stat["custom_json"], len(ops_stream))
async def stream(): try: stm = Steem(node=[ "https://api.steemit.com", "https://steemd.minnowsupportproject.org", "https://anyx.io" ]) chain = Blockchain(stm, "head") for tx in chain.stream(['custom_json']): if tx['id'] == 'sm_sell_cards': user_perm_posting = "" user_perm_active = "" try: user_perm_posting = tx['required_posting_auths'][0] except: pass try: user_perm_active = tx['required_auths'][0] except: pass json_data = json.loads(tx['json']) loop.create_task( process(json_data, user_perm_posting, user_perm_active)) elif tx['id'] == 'sm_update_price': market_id_list = json.loads(tx['json'])['ids'] card_uid_dict = [] for market_id in market_id_list: time.sleep(2) try: response_json = requests.get( f"https://steemmonsters.com/market/status?id={market_id}" ).json() card_uid = response_json['cards'][0]['uid'] card_uid_dict.append(card_uid) except: pass user_perm_posting = "" user_perm_active = "" try: user_perm_posting = tx['required_posting_auths'][0] except: pass try: user_perm_active = tx['required_auths'][0] except: pass loop.create_task( process(card_uid_dict, user_perm_posting, user_perm_active)) await asyncio.sleep(0) except: print(traceback.format_exc())
def anyx(): SV = os.environ.get('SV') di = { '16': .55, '15': .03, '5': .42, '49': .45, '27': .42, '38': .42, '70': .50, '71': .45, '72': .55, '73': .45, '74': .4, } dic = {'16': .7, '5': .50, '49': .7, '27': .56, '38': .68} stm = Steem(node="https://anyx.io", keys=SV) acc = Account('sourovafrin', steem_instance=stm) bo = Blockchain(stm, 'head') print("started anyx") for detail in bo.stream(['custom_json']): try: if detail['id'] == 'sm_sell_cards': for i in ast.literal_eval(detail['json']): lin = requests.get( "https://steemmonsters.com/cards/find?ids=" + i['cards'][0]).json() for ii in lin: id = ii['market_id'] idd = str(ii['card_detail_id']) price = float(ii['buy_price']) if int(ii['edition']) == 1 and price <= di[idd]: acc.transfer( 'svirus', 2, 'SBD', "sm_market_purchase:" + id + ":sourovafrin") elif int(ii['edition']) == 0 and price <= dic[idd]: acc.transfer( 'svirus', 2, 'SBD', "sm_market_purchase:" + id + ":sourovafrin") except Exception as e: print("Error found anyx: {}".format(e))
def main(conn, logger, debug=False): # Main Loop s = Steem(node=nodes) theChain = Blockchain(steem_instance=s, mode="head") Start_block_num = read_blocknum_fromfile("virtual_ops_block.txt") if not Start_block_num: Start_block_num = theChain.get_current_block_num() # print('Start For loop') try: for block in theChain.stream(only_virtual_ops=True, start=Start_block_num): print("Try add to DB") addblock_to_db(block, conn, logger) except Exception as e: return
def main(conn, logger, debug=False): s = Steem(node=nodes) theChain = Blockchain(steem_instance=s, mode="head") # print('Start For loop') Start_block_num = read_blocknum_fromfile("ops_block.txt") if not Start_block_num: Start_block_num = theChain.get_current_block_num() try: for block in theChain.stream(start=Start_block_num): # print("Run Again") # print(f"{block['block_num']}") yay = addblock_to_db(block, conn, logger) # print("Got out ok") except Exception as e: logger.error(f"Error in Stream: {e}") return
def test_stream(self): bts = self.bts start = self.start stop = self.stop b = Blockchain(steem_instance=bts) ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=start, stop=stop): ops_stream.append(op) self.assertTrue(len(ops_stream) > 0) ops_raw_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, raw_ops=True, start=start, stop=stop): ops_raw_stream.append(op) self.assertTrue(len(ops_raw_stream) > 0) only_ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=start, stop=stop, only_ops=True): only_ops_stream.append(op) self.assertTrue(len(only_ops_stream) > 0) only_ops_raw_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, raw_ops=True, start=start, stop=stop, only_ops=True): only_ops_raw_stream.append(op) self.assertTrue(len(only_ops_raw_stream) > 0) op_stat = b.ops_statistics(start=start, stop=stop) op_stat2 = {"transfer": 0, "vote": 0} for op in ops_stream: self.assertIn(op["type"], opNames) op_stat2[op["type"]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat2["transfer"]) self.assertEqual(op_stat["vote"], op_stat2["vote"]) op_stat3 = {"transfer": 0, "vote": 0} for op in ops_raw_stream: self.assertIn(op["op"][0], opNames) op_stat3[op["op"][0]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat3["transfer"]) self.assertEqual(op_stat["vote"], op_stat3["vote"]) op_stat5 = {"transfer": 0, "vote": 0} for op in only_ops_stream: self.assertIn(op["type"], opNames) op_stat5[op["type"]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat5["transfer"]) self.assertEqual(op_stat["vote"], op_stat5["vote"]) op_stat6 = {"transfer": 0, "vote": 0} for op in only_ops_raw_stream: self.assertIn(op["op"][0], opNames) op_stat6[op["op"][0]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat6["transfer"]) self.assertEqual(op_stat["vote"], op_stat6["vote"]) ops_blocks = [] for op in b.blocks(start=start, stop=stop): ops_blocks.append(op) op_stat4 = {"transfer": 0, "vote": 0} self.assertTrue(len(ops_blocks) > 0) for block in ops_blocks: for tran in block["transactions"]: for op in tran['operations']: if isinstance(op, list) and op[0] in opNames: op_stat4[op[0]] += 1 elif isinstance(op, dict): op_type = op["type"] if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation": op_type = op_type[:-10] if op_type in opNames: op_stat4[op_type] += 1 self.assertTrue(block.identifier >= start) self.assertTrue(block.identifier <= stop) self.assertEqual(op_stat["transfer"], op_stat4["transfer"]) self.assertEqual(op_stat["vote"], op_stat4["vote"]) ops_blocks = [] for op in b.blocks(): ops_blocks.append(op) break self.assertTrue(len(ops_blocks) == 1)
class Distribubot: def __init__(self, config, data_file, hived_instance): self.config = config self.data_file = data_file self.hive = hived_instance self.token_config = {} # add log stats self.log_data = { "start_time": 0, "last_block_num": None, "new_commands": 0, "stop_block_num": 0, "stop_block_num": 0, "time_for_blocks": 0 } config_cnt = 0 necessary_fields = [ "token_account", "symbol", "min_token_in_wallet", "comment_command", "token_memo", "reply", "sucess_reply_body", "fail_reply_body", "no_token_left_body", "user_can_specify_amount", "usage_upvote_percentage", "no_token_left_for_today", "token_in_wallet_for_each_outgoing_token", "maximum_amount_per_comment", "count_only_staked_token" ] self.token_config = check_config(self.config["config"], necessary_fields, self.hive) self.blockchain = Blockchain(mode='head', blockchain_instance=self.hive) def run(self, start_block, stop_block): self.hive.wallet.unlock(self.config["wallet_password"]) self.blockchain = Blockchain(mode='head', blockchain_instance=self.hive) current_block = self.blockchain.get_current_block_num() if stop_block is None or stop_block > current_block: stop_block = current_block if start_block is None: start_block = current_block last_block_num = current_block - 1 else: last_block_num = start_block - 1 self.log_data["start_block_num"] = start_block for op in self.blockchain.stream(start=start_block, stop=stop_block, opNames=["comment"]): self.log_data = print_block_log(self.log_data, op, self.config["print_log_at_block"]) last_block_num = op["block_num"] if op["type"] == "comment": token = None for key in self.token_config: if op["body"].find( self.token_config[key]["comment_command"]) >= 0: token = key if token is None: continue if op["author"] == self.token_config[token]["token_account"]: continue cnt = 0 c_comment = None c_parent = None authorperm = construct_authorperm(op) use_tags_api = True while c_comment is None and cnt < 10: cnt += 1 try: c_comment = Comment(authorperm, use_tags_api=use_tags_api, blockchain_instance=self.hive) c_comment.refresh() except: if cnt > 5: use_tags_api = False nodelist = NodeList() nodelist.update_nodes() self.hive = Hive(node=nodelist.get_hive_nodes(), num_retries=5, call_num_retries=3, timeout=15) time.sleep(3) if cnt == 10 or c_comment is None: logger.warn("Could not read %s/%s" % (op["author"], op["permlink"])) continue if 'depth' in c_comment: if c_comment['depth'] == 0: continue else: if c_comment["parent_author"] == '': continue if abs((c_comment["created"] - op['timestamp']).total_seconds()) > 9.0: logger.warn("Skip %s, as edited" % c_comment["authorperm"]) continue already_voted = False if self.token_config[token]["upvote_token_receiver"]: parent_identifier = construct_authorperm( c_comment["parent_author"], c_comment["parent_permlink"]) c_parent = Comment(parent_identifier, blockchain_instance=self.hive) for v in c_parent.get_votes(raw_data=True): if self.token_config[token]["token_account"] == v[ "voter"]: already_voted = True else: for v in c_comment.get_votes(raw_data=True): if self.token_config[token]["token_account"] == v[ "voter"]: already_voted = True if already_voted: continue already_replied = None cnt = 0 if self.token_config[token]["usage_upvote_percentage"] == 0: while already_replied is None and cnt < 5: cnt += 1 try: already_replied = False for r in c_comment.get_all_replies(): if r["author"] == self.token_config[token][ "token_account"]: already_replied = True except: already_replied = None self.hive.rpc.next() if already_replied is None: already_replied = False for r in c_comment.get_all_replies(): if r["author"] == self.token_config[token][ "token_account"]: already_replied = True if already_replied: continue muting_acc = Account(self.token_config[token]["token_account"], blockchain_instance=self.hive) blocked_accounts = muting_acc.get_mutings() if c_comment["author"] in blocked_accounts: logger.info("%s is blocked" % c_comment["author"]) continue # Load bot token balance bot_wallet = Wallet(self.token_config[token]["token_account"], blockchain_instance=self.hive) symbol = bot_wallet.get_token( self.token_config[token]["symbol"]) # parse amount when user_can_specify_amount is true amount = self.token_config[token]["default_amount"] if self.token_config[token]["user_can_specify_amount"]: start_index = c_comment["body"].find( self.token_config[token]["comment_command"]) stop_index = c_comment["body"][start_index:].find("\n") if stop_index >= 0: command = c_comment["body"][start_index + 1:start_index + stop_index] else: command = c_comment["body"][start_index + 1:] command_args = command.replace(' ', ' ').split(" ")[1:] if len(command_args) > 0: try: amount = float(command_args[0]) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.warn("%s - %s - %s" % (str(exc_type), str(fname), str(exc_tb.tb_lineno))) logger.info("Could not parse amount") if self.token_config[token][ "maximum_amount_per_comment"] and amount > self.token_config[ token]["maximum_amount_per_comment"]: amount = self.token_config[token][ "maximum_amount_per_comment"] if not self.config["no_broadcast"] and self.hive.wallet.locked( ): self.hive.wallet.unlock(self.config["wallet_password"]) self.log_data["new_commands"] += 1 wallet = Wallet(c_comment["author"], blockchain_instance=self.hive) token_in_wallet = wallet.get_token( self.token_config[token]["symbol"]) balance = 0 if token_in_wallet is not None: logger.info(token_in_wallet) if self.token_config[token]["count_only_staked_token"]: balance = 0 else: balance = float(token_in_wallet["balance"]) if "stake" in token_in_wallet: balance += float(token_in_wallet['stake']) if 'delegationsIn' in token_in_wallet and float( token_in_wallet['delegationsIn']) > 0: balance += float(token_in_wallet['delegationsIn']) if 'pendingUnstake' in token_in_wallet and float( token_in_wallet['pendingUnstake']) > 0: balance += float(token_in_wallet['pendingUnstake']) if balance > self.token_config[token][ "min_token_in_wallet"]: if self.token_config[token][ "token_in_wallet_for_each_outgoing_token"] > 0: max_token_to_give = int( balance / self.token_config[token] ["token_in_wallet_for_each_outgoing_token"]) else: max_token_to_give = self.token_config[token][ "maximum_amount_per_comment"] else: max_token_to_give = 0 else: max_token_to_give = 0 logger.info("token to give for %s: %f" % (c_comment["author"], max_token_to_give)) db_data = read_data(self.data_file) if "accounts" in db_data and c_comment["author"] in db_data[ "accounts"] and token in db_data["accounts"][ c_comment["author"]]: if db_data["accounts"][c_comment["author"]][token][ "last_date"] == date.today( ) and self.token_config[token][ "token_in_wallet_for_each_outgoing_token"] > 0: max_token_to_give = max_token_to_give - db_data[ "accounts"][c_comment["author"]][token]["amount"] if amount > max_token_to_give: amount = max_token_to_give if amount > self.token_config[token][ "maximum_amount_per_comment"]: amount = self.token_config[token][ "maximum_amount_per_comment"] if token_in_wallet is None or balance < self.token_config[ token]["min_token_in_wallet"]: reply_body = self.token_config[token]["fail_reply_body"] elif max_token_to_give < 1: reply_body = self.token_config[token][ "no_token_left_for_today"] elif c_comment["parent_author"] == c_comment["author"]: reply_body = "You cannot sent token to yourself." elif float(symbol["balance"]) < amount: reply_body = self.token_config[token]["no_token_left_body"] else: if "{}" in self.token_config[token]["sucess_reply_body"]: reply_body = (self.token_config[token] ["sucess_reply_body"]).format( c_comment["parent_author"]) else: reply_body = self.token_config[token][ "sucess_reply_body"] if "{}" in self.token_config[token]["token_memo"]: token_memo = ( self.token_config[token]["token_memo"]).format( c_comment["author"]) else: token_memo = self.token_config[token]["token_memo"] sendwallet = Wallet( self.token_config[token]["token_account"], blockchain_instance=self.hive) try: logger.info( "Sending %.2f %s to %s" % (amount, self.token_config[token]["symbol"], c_comment["parent_author"])) sendwallet.transfer(c_comment["parent_author"], amount, self.token_config[token]["symbol"], token_memo) if "accounts" in db_data: accounts = db_data["accounts"] else: accounts = {} if c_comment["author"] not in accounts: accounts[c_comment["author"]] = {} accounts[c_comment["author"]][token] = { "last_date": date.today(), "n_comments": 1, "amount": amount } elif token not in accounts[c_comment["author"]]: accounts[c_comment["author"]][token] = { "last_date": date.today(), "n_comments": 1, "amount": amount } else: if accounts[c_comment["author"]][token][ "last_date"] < date.today(): accounts[c_comment["author"]][token] = { "last_date": date.today(), "n_comments": 1, "amount": amount } else: accounts[c_comment["author"]][token][ "n_comments"] += 1 accounts[c_comment["author"]][token][ "amount"] += amount store_data(self.data_file, "accounts", accounts) logger.info( "%s - %s" % (c_comment["author"], str(accounts[c_comment["author"]][token]))) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.warn( "%s - %s - %s" % (str(exc_type), str(fname), str(exc_tb.tb_lineno))) logger.warn("Could not send %s token" % self.token_config[token]["symbol"]) continue reply_identifier = construct_authorperm( c_comment["parent_author"], c_comment["parent_permlink"]) if self.config["no_broadcast"]: logger.info("%s" % reply_body) else: try: self.hive.post( "", reply_body, author=self.token_config[token]["token_account"], reply_identifier=reply_identifier) if self.token_config[token][ "usage_upvote_percentage"] <= 0: time.sleep(5) self.hive.post( "", "Command accepted!", author=self.token_config[token] ["token_account"], reply_identifier=c_comment["authorperm"]) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.warn( "%s - %s - %s" % (str(exc_type), str(fname), str(exc_tb.tb_lineno))) logger.warn("Could not reply to post") continue if self.token_config[token]["usage_upvote_percentage"] > 0: time.sleep(5) upvote_percentge = self.token_config[token][ "usage_upvote_percentage"] if self.token_config[token]["scale_upvote_weight"]: upvote_percentge = upvote_percentge * amount / self.token_config[ token]["maximum_amount_per_comment"] print("Upvote with %.2f %%" % upvote_percentge) if self.token_config[token]["upvote_token_receiver"]: if c_parent is None: c_parent = Comment( parent_identifier, blockchain_instance=self.hive) try: c_parent.upvote(upvote_percentge, voter=self.token_config[token] ["token_account"]) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.warn("%s - %s - %s" % (str(exc_type), str(fname), str(exc_tb.tb_lineno))) logger.warn("Could not upvote comment") else: try: c_comment.upvote(upvote_percentge, voter=self.token_config[token] ["token_account"]) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] logger.warn("%s - %s - %s" % (str(exc_type), str(fname), str(exc_tb.tb_lineno))) logger.warn("Could not upvote comment") time.sleep(4) return last_block_num
from beem.account import Account from beem.blockchain import Blockchain from beem.comment import Comment from beem.utils import construct_authorperm from voteleader import db, voter, wif nodes_urls = [ "https://api.hive.blog", "https://api.deathwing.me", "https://hived.privex.io", ] hive = Hive(node=nodes_urls, keys=wif) blockchain = Blockchain(blockchain_instance=hive) stream = blockchain.stream(opNames=["comment"], raw_ops=False, threading=True, thread_num=4) def tally(author): today = datetime.now(timezone.utc) one_week = today - timedelta(days=7) acc = Account(author, blockchain_instance=hive) blog = acc.get_blog() return 1 + sum(post["created"] >= one_week for post in blog) def monitor(): table = db.load_table("leaderboard") vote_table = db["vote_history"] print("[Monitor Starting up...]")
import sys from datetime import timedelta import time import io from beem.blockchain import Blockchain from beem.utils import parse_time import logging log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) class DemoBot(object): def vote(self, vote_event): w = vote_event["weight"] if w > 0: print("Vote by", vote_event["voter"], "for", vote_event["author"]) else: if w < 0: print("Downvote by", vote_event["voter"], "for", vote_event["author"]) else: print("(Down)vote by", vote_event["voter"], "for", vote_event["author"], "CANCELED") if __name__ == "__main__": tb = DemoBot() blockchain = Blockchain() for vote in blockchain.stream(opNames=["vote"]): tb.vote(vote)
latest_update_block = b.get_estimated_block_num(latest_update) else: latest_update_block = start_block print("latest update %s - %d to %d" % (str(latest_update), latest_update_block, stop_block)) start_block = max([latest_update_block, start_block]) + 1 if stop_block > start_block + 6000: stop_block = start_block + 6000 cnt = 0 updated_accounts = [] posts_dict = {} changed_member_data = [] for ops in b.stream(start=start_block, stop=stop_block, opNames=["comment"], max_batch_size=max_batch_size, threading=threading, thread_num=8): #print(ops) timestamp = ops["timestamp"] # timestamp = timestamp.replace(tzinfo=None) # continue if ops["author"] not in member_accounts: continue if ops["block_num"] <= latest_update_block: continue if ops["block_num"] - last_block_print > 50: last_block_print = ops["block_num"] print("blocks left %d - post found: %d" % (ops["block_num"] - stop_block, len(posts_dict))) authorperm = construct_authorperm(ops)
flagged_posts = [] if last_streamed_block == 0: start_block = b.get_current_block_num() - int(201600) else: start_block = last_streamed_block + 1 stop_block = b.get_current_block_num() last_block_print = start_block cnt = 0 updated_accounts = [] posts_dict = {} changed_member_data = [] ops = None for ops in b.stream(start=start_block, stop=stop_block, opNames=["comment", "transfer", "vote", "custom_json"], max_batch_size=max_batch_size, threading=threading, thread_num=8): #print(ops) timestamp = ops["timestamp"] # timestamp = timestamp.replace(tzinfo=None) # continue last_streamed_block = ops["block_num"] if ops["type"] == "transfer" and ops["to"] == "rewarding": authorperm = ops["memo"].split(",")[0] command = ",".join(ops["memo"].split(",")[1:]) commandsTrx.add({ "authorperm": authorperm, "command": command, "account": ops["from"], "valid": True,
def s_load_binary(file_obj): '''load contents from file_obj, returning a generator that yields one element at a time''' for line in file_obj: elt = loads(unhexlify(line[:-1])) yield elt if __name__ == "__main__": blockchain = Blockchain() threading = True thread_num = 8 cur_block = blockchain.get_current_block() stop = cur_block.identifier startdate = cur_block.time() - timedelta(seconds=3600) start = blockchain.get_estimated_block_num(startdate, accurate=True) outf = gzip.open('blocks1.pkl', 'w') blocks = 0 for block in blockchain.stream(opNames=[], start=start, stop=stop, threading=threading, thread_num=thread_num): s_dump_binary(block, outf) blocks = blocks + 1 if blocks % 200 == 0: print(blocks, "blocks streamed") outf.close() for block in s_load_binary(gzip.open('blocks1.pkl')): print(block)
next_account_id = accountTrxStorage.get_latest_index() + 1 if end_block_round > end_block: end_block_round = end_block data_available = False db_data = [] db_account = [] db_follow = [] accounts = accountTrxStorage.get_accounts() last_block_num = None new_accounts_found = 0 new_follow_found = 0 last_trx_id = '0' * 40 op_num = 0 print("start_streaming") for op in b.stream(start=int(start_block), stop=int(end_block_round), opNames=["account_create", "account_create_with_delegation", "create_claimed_account", "pow", "pow2", "custom_json"], max_batch_size=max_batch_size, threading=threading, thread_num=8): block_num = op["block_num"] # print(block_num) if last_block_num is None: new_accounts_found = 0 new_follow_found = 0 start_time = time.time() last_block_num = block_num if op["trx_id"] == last_trx_id: op_num += 1 else: op_num = 0 if "trx_num" in op: trx_num = op["trx_num"] else: trx_num = 0
transaction.update(field_updates=fields, reference=creator_doc_ref) transaction = db.transaction() pref = global_preferences.get().to_dict() last_block_num_synced = pref['last_block_num_synced'] if pref else 26256743 - 1 # HF20 -1 last_block_num = last_block_num_synced + 1 print("Let's sync with blockchain...") for op in blockchain.stream( # opNames=[ # 'claim_account', # 'create_claimed_account', # 'account_create', # ], # start=26256743, # first block of HF20 # start=26265398, # first block after HF20 with `create_claimed_account` op # start=26289186, # first block after HF20 with `account_create` op start=last_block_num_synced + 1, max_batch_size=50, ): op["timestamp"] = op["timestamp"].__str__() if last_block_num < op['block_num']: # that means that all ops from last_block are already synced last_block_num_synced = op['block_num'] - 1 current_block_num = blockchain.get_current_block_num() transaction.set(global_preferences, { "last_block_num_synced": last_block_num_synced, "current_block_num": current_block_num, "blocks_behind": current_block_num - op['block_num'], "last_trx_id_synced": op["trx_id"],
import dataset from beem import Steem from beem.blockchain import Blockchain # Ask for user ID hive_id = input("Hive User ID: ") # hive_id = yournamehere # if you want to hard code it. hive = Steem(node='https://anyx.io') db = dataset.connect('sqlite:///mydatabase.db') # System Variables blockchain = Blockchain(steem_instance=hive) stream = blockchain.stream(opNames=['transfer'], raw_ops=False, threading=True, thread_num=4) table = db[hive_id] # parse json data to SQL insert def update_db(post): try: table.insert(dict(post)) db.commit() except Exception as e: print(f'[Error: {e} moving on]') db.rollback() def monitor():
from __future__ import print_function import sys from datetime import timedelta import time import io from beem.blockchain import Blockchain from beem.utils import parse_time import logging log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) class DemoBot(object): def comment(self, comment_event): print('Comment by {} on post {} by {}:'.format( comment_event['author'], comment_event['parent_permlink'], comment_event['parent_author'])) print(comment_event['body']) print() if __name__ == "__main__": tb = DemoBot() blockchain = Blockchain() for vote in blockchain.stream(opNames=["comment"]): tb.comment(vote)
except exceptions.ContentDoesNotExistsException: print("Could not find Comment: %s" % (authorperm)) al = list() if not vote_event["voter"] in self.looked_up: al.append(vote_event["voter"]) self.looked_up.add(vote_event["voter"]) if not vote_event["author"] in self.looked_up: al.append(vote_event["author"]) self.looked_up.add(vote_event["author"]) if len(al) > 0: lookup_accounts(al) if __name__ == "__main__": wtw = WatchingTheWatchers() tb = WatchingTheWatchersBot(wtw) blockchain = Blockchain() threading = True thread_num = 16 cur_block = blockchain.get_current_block() stop = cur_block.identifier startdate = cur_block.time() - timedelta(days=1) start = blockchain.get_estimated_block_num(startdate, accurate=True) for vote in blockchain.stream(opNames=["vote"], start=start, stop=stop, threading=threading, thread_num=thread_num): tb.vote(vote) wtw.report()