def test_stream_batch(self): bts = self.bts b = Blockchain(steem_instance=bts) ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False): ops_stream.append(op) self.assertTrue(ops_stream[0]["block_num"] >= self.start) self.assertTrue(ops_stream[-1]["block_num"] <= self.stop) op_stat = b.ops_statistics(start=self.start, stop=self.stop) self.assertEqual(op_stat["vote"] + op_stat["transfer"], len(ops_stream)) ops_blocks = [] for op in b.blocks(start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False): ops_blocks.append(op) op_stat4 = {"transfer": 0, "vote": 0} self.assertTrue(len(ops_blocks) > 0) for block in ops_blocks: for tran in block["transactions"]: for op in tran['operations']: if isinstance(op, dict) and "type" in op and "value" in op: op_type = op["type"] if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation": op_type = op_type[:-10] if op_type in opNames: op_stat4[op_type] += 1 elif op[0] in opNames: op_stat4[op[0]] += 1 self.assertTrue(block.identifier >= self.start) self.assertTrue(block.identifier <= self.stop) self.assertEqual(op_stat["transfer"], op_stat4["transfer"]) self.assertEqual(op_stat["vote"], op_stat4["vote"])
def test_stream_threading(self): bts = self.bts b = Blockchain(steem_instance=bts) ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=self.start, stop=self.stop): ops_stream.append(op) self.assertTrue(len(ops_stream) > 0) op_stat = b.ops_statistics(start=self.start, stop=self.stop) ops_blocks = [] for op in b.blocks(start=self.start, stop=self.stop, threading=True, thread_num=8): ops_blocks.append(op) op_stat4 = {"transfer": 0, "vote": 0} self.assertTrue(len(ops_blocks) > 0) for block in ops_blocks: for tran in block["transactions"]: for op in tran['operations']: if op[0] in opNames: op_stat4[op[0]] += 1 self.assertTrue(block.identifier >= self.start) self.assertTrue(block.identifier <= self.stop) self.assertEqual(op_stat["transfer"], op_stat4["transfer"]) self.assertEqual(op_stat["vote"], op_stat4["vote"])
class Steem_node(): def __init__(self): self.stm = Steem() self.blockchain = Blockchain(steem_instance=self.stm, mode="head") self.mailserver = Gmail(60) def run(self): # Fetch full blocks, then inspect each operation. for block in self.blockchain.blocks(): block_num = block.block_num created_at = block.time() print(f'Block: {block_num}') for op in block.operations: # Extract op data if op['type'] == 'transfer_operation': type = 'transfer' subject = 'New transfer' to = op['value']['to'] FROM = op['value']['from'] amount = op['value']['amount'] memo = op['value']['memo'] # Check if account matches, sent mail on True if to == 'steempytutorials': print('Incoming transfer') message = ( f'{created_at}\nBlock: {block_num}\nFrom: {FROM}' + f'\nTo: {to}\nAmount: {amount}\nMemo: {memo}\n') try: self.mailserver.send_email(subject, message, type) print('Mail sent') except Exception as e: print('Failed to sent mail', e)
def test_signing_appbase(self): b = Blockchain(steem_instance=self.bts) st = None for block in b.blocks(start=25304468, stop=25304468): for trx in block.transactions: st = Signed_Transaction(trx.copy()) self.assertTrue(st is not None)
def find_steem_tx(self, tx_data: dict, last_blocks=15) -> dict: """ Used internally to get the transaction ID after a Steem transaction has been broadcasted. See :py:meth:`.send_token` and :py:meth:`.issue_token` for how this is used. :param dict tx_data: Transaction data returned by a beem broadcast operation, must include ``signatures`` :param int last_blocks: Amount of previous blocks to search for the transaction :return dict: Transaction data from the blockchain, see below. **Return format:** .. code-block:: js { transaction_id, ref_block_num, ref_block_prefix, expiration, operations, extensions, signatures, block_num, transaction_num } :return None: If the transaction wasn't found, ``None`` will be returned. """ from beem.blockchain import Blockchain # Code taken/based from @holgern/beem blockchain.py chain = Blockchain(steem_instance=self.steem, mode='head') current_num = chain.get_current_block_num() for block in chain.blocks(start=current_num - last_blocks, stop=current_num + 5): for tx in block.transactions: if sorted(tx["signatures"]) == sorted(tx_data["signatures"]): return tx return None
def test_block_threading(self): bts = self.bts b = Blockchain(steem_instance=bts) blocks_no_threading = [] for block in b.blocks(start=self.start, stop=self.stop, threading=False, thread_num=8): blocks_no_threading.append(block) for n in range(5): blocks = [] for block in b.blocks(start=self.start, stop=self.stop, threading=True, thread_num=8): blocks.append(block) for i in range(min(len(blocks), len(blocks_no_threading))): self.assertEqual(blocks[i]["block_id"], blocks_no_threading[i]["block_id"]) self.assertEqual(len(blocks_no_threading), len(blocks))
def hour_active(chain): nodelist = NodeList() nodelist.update_nodes() if chain=='steem': s = Steem(node=nodelist.get_steem_nodes()) else: s = Steem(node=nodelist.get_hive_nodes()) b=Blockchain(s) bl_=[] bl_num=int(b.get_current_block_num()) bl_num_=bl_num-1250 bl=b.blocks(bl_num_,bl_num) for i in bl: bl_.append(i['transactions']) x=('follower','account','voter','from','author') account=[] new=[] for i in x: acc=re.findall('"'+i+'":"(.+?)"',str(bl_)) for l in acc: if l not in account: account.append(l) if chain=='steem': file=open('active_acc_steem.txt','r') old=file.readlines() file.close() file=open('active_acc_steem.txt','a') for i in account: if i+'\n' not in old: file.write(str(i)+'\n') new.append(i) file.close() else: file=open('active_acc_hive.txt','r') old=file.readlines() file.close() file=open('active_acc_hive.txt','a') for i in account: if i+'\n' not in old: file.write(str(i)+'\n') new.append(i) file.close() print ('one hour accounts '+chain,len(account)) print ('one hour brand new '+chain,len(new)) print ('total ACTIVE accounts '+chain,len(old))
async def main(): blocks = [] hive = Hive(HIVE_NODES) chain = Blockchain(blockchain_instance=hive) print(f"\n [{datetime.utcnow()!s}] Loading last {NUM_BLOCKS} blocks using beem ... \n\n") start_time = time.time_ns() current_num = chain.get_current_block_num() for block in chain.blocks(start=current_num - NUM_BLOCKS, stop=current_num): blocks.append(block) end_time = time.time_ns() print(f"\n [{datetime.utcnow()!s}] Total blocks:", len(blocks), "\n") start_time, end_time = Decimal(start_time), Decimal(end_time) start_secs = start_time / SECS_NS end_secs = end_time / SECS_NS print("Start Time:", dec_round(start_secs, 4), "seconds") print("End Time:", dec_round(end_secs, 4), "seconds\n") print("Total Time:", dec_round(end_secs - start_secs, 4), "seconds\n")
def find_steem_tx(self, tx_data, last_blocks=15) -> Optional[dict]: """ Used internally to get the transaction ID after a transaction has been broadcasted :param dict tx_data: Transaction data returned by a beem broadcast operation, must include 'signatures' :param int last_blocks: Amount of previous blocks to search for the transaction :return dict: Transaction data from the blockchain {transaction_id, ref_block_num, ref_block_prefix, expiration, operations, extensions, signatures, block_num, transaction_num} :return None: If the transaction wasn't found, None will be returned. """ # Code taken/based from @holgern/beem blockchain.py chain = Blockchain(steem_instance=self.rpc, mode='head') current_num = chain.get_current_block_num() for block in chain.blocks(start=current_num - last_blocks, stop=current_num + 5): for tx in block.transactions: if sorted(tx["signatures"]) == sorted(tx_data["signatures"]): return tx return None
def benchmark_node(node, how_many_minutes=10, how_many_seconds=30): block_count = 0 history_count = 0 access_time = 0 follow_time = 0 blockchain_version = u'0.0.0' successful = True error_msg = None start_total = timer() max_batch_size = None threading = False thread_num = 16 authorpermvoter = u"@gtg/steem-pressure-4-need-for-speed|gandalf" [author, permlink, voter] = resolve_authorpermvoter(authorpermvoter) authorperm = construct_authorperm(author, permlink) last_block_id = 19273700 try: stm = Steem(node=node, num_retries=3, num_retries_call=3, timeout=30) blockchain = Blockchain(steem_instance=stm) blockchain_version = stm.get_blockchain_version() last_block = Block(last_block_id, steem_instance=stm) stopTime = last_block.time() + timedelta(seconds=how_many_minutes * 60) total_transaction = 0 start = timer() for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num): block_no = entry.identifier block_count += 1 if "block" in entry: trxs = entry["block"]["transactions"] else: trxs = entry["transactions"] for tx in trxs: for op in tx["operations"]: total_transaction += 1 if "block" in entry: block_time = parse_time(entry["block"]["timestamp"]) else: block_time = parse_time(entry["timestamp"]) if block_time > stopTime: last_block_id = block_no break if timer() - start > how_many_seconds or quit_thread: break except NumRetriesReached: error_msg = 'NumRetriesReached' block_count = -1 except KeyboardInterrupt: error_msg = 'KeyboardInterrupt' # quit = True except Exception as e: error_msg = str(e) block_count = -1 try: stm = Steem(node=node, num_retries=3, num_retries_call=3, timeout=30) account = Account("gtg", steem_instance=stm) blockchain_version = stm.get_blockchain_version() start = timer() for acc_op in account.history_reverse(batch_size=100): history_count += 1 if timer() - start > how_many_seconds or quit_thread: break except NumRetriesReached: error_msg = 'NumRetriesReached' history_count = -1 successful = False except KeyboardInterrupt: error_msg = 'KeyboardInterrupt' history_count = -1 successful = False # quit = True except Exception as e: error_msg = str(e) history_count = -1 successful = False try: stm = Steem(node=node, num_retries=3, num_retries_call=3, timeout=30) account = Account("gtg", steem_instance=stm) blockchain_version = stm.get_blockchain_version() start = timer() Vote(authorpermvoter, steem_instance=stm) stop = timer() vote_time = stop - start start = timer() Comment(authorperm, steem_instance=stm) stop = timer() comment_time = stop - start start = timer() Account(author, steem_instance=stm) stop = timer() account_time = stop - start start = timer() account.get_followers() stop = timer() follow_time = stop - start access_time = (vote_time + comment_time + account_time + follow_time) / 4.0 except NumRetriesReached: error_msg = 'NumRetriesReached' access_time = -1 except KeyboardInterrupt: error_msg = 'KeyboardInterrupt' # quit = True except Exception as e: error_msg = str(e) access_time = -1 return {'successful': successful, 'node': node, 'error': error_msg, 'total_duration': timer() - start_total, 'block_count': block_count, 'history_count': history_count, 'access_time': access_time, 'follow_time': follow_time, 'version': blockchain_version}
year = int(sys.argv[1]) month = int(sys.argv[2]) day = int(sys.argv[3]) nl = NodeList() s = Steem(node=nl.get_nodes(appbase=True, normal=False)) b = Blockchain(steem_instance=s) start_date = datetime(year, month, day) end_date = start_date + timedelta(days=1) start_block = b.get_estimated_block_num(start_date) end_block = b.get_estimated_block_num(end_date) - 1 m = MongoStorage(db_name="steem", host='172.18.0.3', port=27017, user='', passwd='') m.ensure_indexes() for block in b.blocks(start=start_block, stop=end_block, max_batch_size=BATCH_SIZE): sys.stdout.write("%s - %s - %s\r" % (start_block, block.block_num, end_block)) for trx in block.transactions: if "'vote_operation'" not in str(trx): continue m.Transactions.insert(trx)
def test_stream(self): bts = self.bts start = self.start stop = self.stop b = Blockchain(steem_instance=bts) ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=start, stop=stop): ops_stream.append(op) self.assertTrue(len(ops_stream) > 0) ops_raw_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, raw_ops=True, start=start, stop=stop): ops_raw_stream.append(op) self.assertTrue(len(ops_raw_stream) > 0) only_ops_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, start=start, stop=stop, only_ops=True): only_ops_stream.append(op) self.assertTrue(len(only_ops_stream) > 0) only_ops_raw_stream = [] opNames = ["transfer", "vote"] for op in b.stream(opNames=opNames, raw_ops=True, start=start, stop=stop, only_ops=True): only_ops_raw_stream.append(op) self.assertTrue(len(only_ops_raw_stream) > 0) op_stat = b.ops_statistics(start=start, stop=stop) op_stat2 = {"transfer": 0, "vote": 0} for op in ops_stream: self.assertIn(op["type"], opNames) op_stat2[op["type"]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat2["transfer"]) self.assertEqual(op_stat["vote"], op_stat2["vote"]) op_stat3 = {"transfer": 0, "vote": 0} for op in ops_raw_stream: self.assertIn(op["op"][0], opNames) op_stat3[op["op"][0]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat3["transfer"]) self.assertEqual(op_stat["vote"], op_stat3["vote"]) op_stat5 = {"transfer": 0, "vote": 0} for op in only_ops_stream: self.assertIn(op["type"], opNames) op_stat5[op["type"]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat5["transfer"]) self.assertEqual(op_stat["vote"], op_stat5["vote"]) op_stat6 = {"transfer": 0, "vote": 0} for op in only_ops_raw_stream: self.assertIn(op["op"][0], opNames) op_stat6[op["op"][0]] += 1 self.assertTrue(op["block_num"] >= start) self.assertTrue(op["block_num"] <= stop) self.assertEqual(op_stat["transfer"], op_stat6["transfer"]) self.assertEqual(op_stat["vote"], op_stat6["vote"]) ops_blocks = [] for op in b.blocks(start=start, stop=stop): ops_blocks.append(op) op_stat4 = {"transfer": 0, "vote": 0} self.assertTrue(len(ops_blocks) > 0) for block in ops_blocks: for tran in block["transactions"]: for op in tran['operations']: if isinstance(op, list) and op[0] in opNames: op_stat4[op[0]] += 1 elif isinstance(op, dict): op_type = op["type"] if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation": op_type = op_type[:-10] if op_type in opNames: op_stat4[op_type] += 1 self.assertTrue(block.identifier >= start) self.assertTrue(block.identifier <= stop) self.assertEqual(op_stat["transfer"], op_stat4["transfer"]) self.assertEqual(op_stat["vote"], op_stat4["vote"]) ops_blocks = [] for op in b.blocks(): ops_blocks.append(op) break self.assertTrue(len(ops_blocks) == 1)
class LatestPostManager(): def __init__(self): self.mysql_con = config.get_connection() if self.mysql_con is None: print( "[INFO] Can't start Latest Post Manager because of an mysql database error!" ) return self.mysql_cursor = self.mysql_con.cursor() self.query = "INSERT INTO latest_posts (author, permlink, category, timestamp) VALUES (%s, %s, %s, %s);" self.chain = Blockchain( blockchain_instance=Hive()) #node=conf.HIVE_NODES[5] self.run_thread = Thread(target=self.run) self.run_thread.name = 'Get & Categorize Posts' self.run_thread.daemon = True self.run_thread.start() def enter_posts_by_block(self, block): for op in block.operations: if op['type'] == 'comment_operation': action = op['value'] if action['parent_author'] == '': # found post --> Categorize _input = network.WordEmbedding.vectorize_text( config.statics.Word2Vec, html=action['body'], text=action['title'] + ". ") if _input is None: # to short or error continue # Categorize _output = config.statics.TextCNN(_input).cpu() # Enter in Mysql str_arr = ' '.join(map(str, _output.data[0].tolist())) result = database.commit_query( self.query, (action['author'], action['permlink'], str_arr, block["timestamp"].strftime("%d.%m.%YT%H:%M:%S"))) if result == -1: print("[WARNING] Can't enter post in database!") time.sleep(5) def clean_up(self): # TODO: Implement sorting order by timestamp query = "SELECT timestamp FROM latest_posts;" for item in database.read_query(query, None): timestamp = datetime.strptime(item[0], "%d.%m.%YT%H:%M:%S") if timestamp < (datetime.utcnow() - timedelta(days=5)): result = database.commit_query( "DELETE FROM latest_posts WHERE timestamp=%s;", (item[0], )) def run(self): current_num = self.chain.get_current_block_num() - int( 60 * 60 * 24 * 3 / 3 ) # Get all posts from the last 3 days because it takes a long time to get all and when it finished, the clean_up begins while True: if current_num < self.chain.get_current_block_num(): # if block is available try: for block in self.chain.blocks(start=current_num, stop=current_num): self.enter_posts_by_block(block) except: pass current_num += 1 if current_num % 100 == 0: self.clean_up() else: time.sleep(0.5) else: # wait until new block is created # Using time for cleanup self.clean_up()
def benchmark_node_blocks(node, num_retries=10, num_retries_call=10, timeout=60, how_many_seconds=30): block_count = 0 sucessfull = False error_msg = None start_total = timer() start_time = timer() max_batch_size = None threading = False thread_num = 16 last_block_id = 19273700 try: stm = Steem(node=node, num_retries=num_retries, num_retries_call=num_retries_call, timeout=timeout) blockchain = Blockchain(steem_instance=stm) last_block_id = int(blockchain.get_current_block_num() * 0.75) last_block = Block(last_block_id, steem_instance=stm) total_transaction = 0 start_time = timer() for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num): block_no = entry.identifier block_count += 1 if not sucessfull: sucessfull = True if "block" in entry: trxs = entry["block"]["transactions"] else: trxs = entry["transactions"] for tx in trxs: for op in tx["operations"]: total_transaction += 1 if "block" in entry: block_time = (entry["block"]["timestamp"]) else: block_time = (entry["timestamp"]) if timer() - start_time > how_many_seconds or quit_thread: break except NumRetriesReached: error_msg = 'NumRetriesReached' sucessfull = False except KeyboardInterrupt: error_msg = 'KeyboardInterrupt' # quit = True except Exception as e: error_msg = str(e) sucessfull = False total_duration = float("{0:.2f}".format(timer() - start_time)) return { 'sucessfull': sucessfull, 'node': node, 'error': error_msg, 'total_duration': total_duration, 'count': block_count, 'access_time': None }
year = 2019 month = int(sys.argv[1]) day = int(sys.argv[2]) start_date = datetime(year, month, day) stop_date = start_date + timedelta(days=1) DW_IDs = ['dw-heist', 'drugwars', 'dw-unit', 'dw-upgrade', 'dw-char'] stm = None #Steem(node=['https://api.steemit.com']) b = Blockchain(steem_instance=stm) start_block = b.get_estimated_block_num(start_date) stop_block = b.get_estimated_block_num(stop_date) transactions = [] for blk in b.blocks(start=start_block, stop=stop_block, threading=True): # max_batch_size=50): sys.stdout.write("%s\r" % (blk['timestamp'])) for trx in blk.transactions: keep = False for op in trx['operations']: if op['type'] == 'custom_json_operation' and \ op['value']['id'] in DW_IDs: keep = True if keep: transactions.append(trx) s = shelve.open("transactions_%04d%02d%02d.shelf" % (year, month, day)) s['transactions'] = transactions s.close()
def get_transactions(start_block_num, log_data): config_data = read_config() databaseConnector = config_data["databaseConnector"] custom_json_id = config_data["custom_json_id"] transfer_id = config_data["transfer_id"] current_block = start_block_num - 1 nodes = NodeList() # weights = {'block': 1, 'history': 0.5, 'apicall': 0.5, 'config': 0.5} nodes.update_nodes() stm = Steem(node=["https://api.steemit.com"]) print(stm) b = Blockchain(mode="head", steem_instance=stm) if "stop_block_num" in config_data and config_data["stop_block_num"] > 0: stop_block_num = config_data["stop_block_num"] else: stop_block_num = b.get_current_block_num() if stop_block_num - start_block_num > 1200: stop_block_num = start_block_num + 1200 log_data["stop_block_num"] = stop_block_num db = dataset.connect(databaseConnector) table = db['status'] table.upsert({"id": 1, "tracker_stop_block_num": stop_block_num}, ["id"]) table = db["blocks"] data = table.find_one(order_by='-block_num') if data is None: latest_block_num = start_block_num - 1 else: latest_block_num = data["block_num"] table = db["virtualops"] for block in b.blocks(start=start_block_num, stop=stop_block_num, threading=False, thread_num=8): if "transactions" in block: trx = block["transactions"] else: trx = [block] block_num = 0 trx_id = "" _id = "" timestamp = "" for trx_nr in range(len(trx)): if "operations" not in trx[trx_nr]: continue for event in trx[trx_nr]["operations"]: if isinstance(event, list): op_type, op = event trx_id = block["transaction_ids"][trx_nr] block_num = block.get("id") timestamp = block.get("timestamp") block_id = block.get("block_id") previous = block.get("previous") elif isinstance(event, dict) and "type" in event and "value" in event: op_type = event["type"] if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation": op_type = op_type[:-10] op = event["value"] trx_id = block["transaction_ids"][trx_nr] block_num = block.get("id") timestamp = block.get("timestamp") block_id = block.get("block_id") previous = block.get("previous") elif "op" in event and isinstance(event["op"], dict) and "type" in event["op"] and "value" in event["op"]: op_type = event["op"]["type"] if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation": op_type = op_type[:-10] op = event["op"]["value"] trx_id = event.get("trx_id") block_num = event.get("block") timestamp = event.get("timestamp") block_id = None previous = None else: op_type, op = event["op"] trx_id = event.get("trx_id") block_num = event.get("block") timestamp = event.get("timestamp") block_id = None previous = None # print(op) log_data = print_block_log(start_block_num, log_data, block_num, timestamp) current_block = block_num date = timestamp.replace(tzinfo=None) if block_num > latest_block_num: table = db["blocks"] table.insert({"block_num": block_num, "timestamp": date, "block_id": block_id, "previous": previous}) latest_block_num = block_num if op_type == "transfer": if op["to"] != "nextcolony": continue user = op["from"] memo = op["memo"] at_symbol_pos = memo.find('@') if at_symbol_pos < 0: continue if memo[:at_symbol_pos] != transfer_id: continue amount = Amount(op["amount"], steem_instance=stm) log_data["new_transfers"] += 1 table = db['transfers'] if table.find_one(trx=trx_id) == None: print ("Writing the transaction into the DB") data = dict(trx=trx_id, user=user, block_num=int(block_num), amount=str(amount), memo=memo, tr_status=0, date=date) table.insert(data) table = db['status'] table.upsert({"id": 1, "latest_block_num": int(block_num)}, ["id"]) elif op_type == "custom_json": if op['id'] != custom_json_id: continue date = timestamp.replace(tzinfo=None) try: json_data = json.loads(op['json']) except: print("Skip json: %s" % str(op['json'])) continue if isinstance(json_data, list) and len(json_data) == 1: json_data = json_data[0] if not isinstance(json_data, dict): print(json_data) print("not a valid json") continue if 'command' not in json_data: print (json_data) print("wrong cusom_json structure") continue if len(json_data['command']) == 0: print(json_data) print("command is empty") continue if len(op['required_posting_auths']) > 0: user = op['required_posting_auths'][0] elif len(op['required_auths']) > 0: user = op['required_auths'][0] else: print("Cannot parse transaction, as user could not be determined!") continue table = db['transactions'] if table.find_one(trx=trx_id) != None: print("Skip trx_id %s at %s" % (trx_id, date)) continue log_data["new_custom_json"] += 1 command = json_data['command'] if "type" not in json_data: continue if json_data["type"] is None: continue json_type = json_data['type'] print("%s: %d:%s - @%s type: %s, command: %s" % (str(date), block_num, trx_id, user, str(json_type), str(command))) if not isinstance(command, dict): print("Skip trx_id %s at %s" % (trx_id, date)) continue if "tr_var1" in command: tr_var1 = unidecode(str(command['tr_var1'])) else: tr_var1 = 0 if "tr_var2" in command: tr_var2 = unidecode(str(command['tr_var2'])) else: tr_var2 = 0 if "tr_var3" in command: tr_var3 = unidecode(str(command['tr_var3'])) else: tr_var3 = 0 if "tr_var4" in command: tr_var4 = unidecode(str(command['tr_var4'])) else: tr_var4 = 0 if "tr_var5" in command: tr_var5 = unidecode(str(command['tr_var5'])) else: tr_var5 = 0 if "tr_var6" in command: tr_var6 = unidecode(str(command['tr_var6'])) else: tr_var6 = 0 if "tr_var7" in command: tr_var7 = unidecode(str(command['tr_var7'])) else: tr_var7 = 0 if "tr_var8" in command: tr_var8 = unidecode(str(command['tr_var8'])) else: tr_var8 = 0 table = db['transactions'] data = dict(trx=trx_id, user=user, tr_type=json_type, block_num=int(block_num), tr_var1=str(tr_var1), tr_var2=str(tr_var2), tr_var3=tr_var3, tr_var4=tr_var4, tr_var5=tr_var5, tr_var6=tr_var6, tr_var7=tr_var7, tr_var8=tr_var8, tr_status=0 ,date=date) table.insert(data) table = db['status'] table.upsert({"id": 1, "latest_block_num": int(block_num)}, ["id"]) return current_block + 1, log_data
last_block_id = 19273700 last_block = Block(last_block_id, steem_instance=stm) startTime = datetime.now() stopTime = last_block.time() + timedelta(seconds=how_many_minutes * 60) ltime = time.time() cnt = 0 total_transaction = 0 start_time = time.time() last_node = blockchain.steem.rpc.url for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num): block_no = entry.identifier if "block" in entry: trxs = entry["block"]["transactions"] else: trxs = entry["transactions"] for tx in trxs: for op in tx["operations"]: total_transaction += 1 if "block" in entry: block_time = parse_time(entry["block"]["timestamp"]) else: block_time = parse_time(entry["timestamp"])
def main(args=None): args = parse_args(args) blockchain = args.blockchain nodelist = NodeList() nodelist.update_nodes(weights={"block": 1}) if blockchain == "hive" or blockchain is None: max_batch_size = 50 threading = False thread_num = 16 block_debug = 1000 nodes = nodelist.get_hive_nodes() blk_inst = Hive(node=nodes, num_retries=3, num_retries_call=3, timeout=30) elif blockchain == "blurt": max_batch_size = None threading = False thread_num = 8 block_debug = 20 nodes = [ "https://api.blurt.blog", "https://rpc.blurtworld.com", "https://rpc.blurtworld.com" ] blk_inst = Blurt(node=nodes, num_retries=3, num_retries_call=3, timeout=30) elif blockchain == "steem": max_batch_size = 50 threading = False thread_num = 16 block_debug = 1000 nodes = nodelist.get_steem_nodes() blk_inst = Steem(node=nodes, num_retries=3, num_retries_call=3, timeout=30) else: raise Exception("Wrong parameter, can be hive, blurt or steem") print(blk_inst) block_count = 0 total_ops = 0 total_virtual_ops = 0 total_trx = 0 blocksperday = 20 * 60 * 24 blockchain = Blockchain(blockchain_instance=blk_inst, ) current_block_num = blockchain.get_current_block_num() last_block_id = current_block_num - blocksperday last_block = Block(last_block_id, blockchain_instance=blk_inst) stopTime = last_block.time() + timedelta(seconds=60 * 60 * 24) start = timer() for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num): if "block" in entry: block_time = parse_time(entry["block"]["timestamp"]) else: block_time = entry["timestamp"] if block_time > stopTime: break block_count += 1 if "block" in entry: trxs = entry["block"]["transactions"] else: trxs = entry["transactions"] for tx in trxs: total_trx += 1 for op in tx["operations"]: total_ops += 1 ops_per_day = total_ops / block_count * blocksperday if block_count % (block_debug) == 0: print("%d blocks remaining... estimated ops per day: %.1f" % (blocksperday - block_count, ops_per_day)) duration = timer() - start stopTime = last_block.time() + timedelta(seconds=60 * 60 * 24) start = timer() for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num, only_virtual_ops=True): block_time = entry["timestamp"] if block_time > stopTime: break for tx in entry["operations"]: for op in tx["op"]: total_virtual_ops += 1 duration = timer() - start print("Received %.2f blocks/s." % (block_count / duration)) print("Bocks: %d, duration %.3f s" % (block_count, duration)) print("Operations per day: %d" % total_ops) print("Trx per day: %d" % total_trx) print("Virtual Operations per day: %d" % total_virtual_ops)
def main(args=None): args = parse_args(args) blockchain = args.blockchain nodelist = NodeList() nodelist.update_nodes(weights={"block": 1}) if blockchain == "hive" or blockchain is None: max_batch_size = 50 threading = False thread_num = 16 block_debug = 1000 nodes = nodelist.get_hive_nodes() blk_inst = Hive(node=nodes, num_retries=3, num_retries_call=3, timeout=30) elif blockchain == "blurt": max_batch_size = None threading = False thread_num = 8 block_debug = 20 nodes = ["https://rpc.blurt.buzz/", "https://api.blurt.blog", "https://rpc.blurtworld.com", "https://rpc.blurtworld.com"] blk_inst = Blurt(node=nodes, num_retries=3, num_retries_call=3, timeout=30) elif blockchain == "steem": max_batch_size = 50 threading = False thread_num = 16 block_debug = 1000 nodes = nodelist.get_steem_nodes() blk_inst = Steem(node=nodes, num_retries=3, num_retries_call=3, timeout=30) else: raise Exception("Wrong parameter, can be hive, blurt or steem") print(blk_inst) block_count = 0 total_ops = 0 total_trx = 0 duration_s = 60 * 60 * 1 blocksperday = int(duration_s / 3) blockchain = Blockchain(blockchain_instance=blk_inst, ) current_block_num = blockchain.get_current_block_num() last_block_id = current_block_num - blocksperday last_block = Block(last_block_id, blockchain_instance=blk_inst) stopTime = last_block.time() + timedelta(seconds=duration_s) start = timer() op_stats = {} for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num): if "block" in entry: block_time = parse_time(entry["block"]["timestamp"]) else: block_time = entry["timestamp"] if block_time > stopTime: break block_count += 1 if "block" in entry: trxs = entry["block"]["transactions"] else: trxs = entry["transactions"] for tx in trxs: total_trx += 1 for op in tx["operations"]: if "_operation" in op["type"]: op_type = op["type"][:-10] else: op_type = op["type"] if op_type in op_stats: op_stats[op_type] += 1 else: op_stats[op_type] = 1 total_ops += 1 ops_per_day = total_ops / block_count * blocksperday if block_count % (block_debug) == 0: print("%d blocks remaining... estimated ops per day: %.1f" % (blocksperday - block_count, ops_per_day)) duration = timer() - start t = PrettyTable(["Type", "Count", "percentage"]) t.align = "l" op_list = [] for o in op_stats: op_list.append({"type": o, "n": op_stats[o], "perc": op_stats[o] / total_ops * 100}) op_list_sorted = sorted(op_list, key=lambda x: x['n'], reverse=True) for op in op_list_sorted: t.add_row([op["type"], op["n"], "%.2f %%" % op["perc"]]) print(t)
import sys import shelve from beem.blockchain import Blockchain from datetime import datetime, timedelta start_date = datetime(2019, 3, 5) stop_date = start_date + timedelta(days=1) DW_IDs = ['dw-heist', 'drugwars', 'dw-unit', 'dw-upgrade', 'dw-char'] b = Blockchain() start_block = b.get_estimated_block_num(start_date) stop_block = b.get_estimated_block_num(stop_date) transactions = [] for blk in b.blocks(start=start_block, stop=stop_block, max_batch_size=50): sys.stdout.write("%s\r" % (blk['timestamp'])) for trx in blk.transactions: keep = False for op in trx['operations']: if op['type'] == 'custom_json_operation' and \ op['value']['id'] in DW_IDs: keep = True if keep: transactions.append(trx) s = shelve.open("transactions.shelf") s['transactions'] = transactions s.close()
from beem.blockchain import Blockchain import time nodes = ["wss://steemd.pevo.science"] threading = True thread_num = 16 stm = beem.steem.Steem(nodes) b = Blockchain(steem_instance=stm) starttime = time.time() dvcount = 0 firstblock = 18581583 while True: lastblock = firstblock + 100 if lastblock > 18610152: lastblock = 18610152 blocks = b.blocks(firstblock, lastblock, threading=threading, thread_num=thread_num) for block in blocks: for t in block["transactions"]: for o in t["operations"]: if o[0] == "vote" and o[1]["weight"] < 0: dvcount = dvcount + 1 if dvcount % 250 == 0: print("Processed", dvcount) firstblock = lastblock if firstblock >= 18610152: break endtime = time.time() print("dvcount = ", dvcount) print("Processing January first of 2018 took", endtime - starttime, "seconds")