def test_stream_batch(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False):
ops_stream.append(op)
self.assertTrue(ops_stream[0]["block_num"] >= self.start)
self.assertTrue(ops_stream[-1]["block_num"] <= self.stop)
op_stat = b.ops_statistics(start=self.start, stop=self.stop)
self.assertEqual(op_stat["vote"] + op_stat["transfer"], len(ops_stream))
ops_blocks = []
for op in b.blocks(start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False):
ops_blocks.append(op)
op_stat4 = {"transfer": 0, "vote": 0}
self.assertTrue(len(ops_blocks) > 0)
for block in ops_blocks:
for tran in block["transactions"]:
for op in tran['operations']:
if isinstance(op, dict) and "type" in op and "value" in op:
op_type = op["type"]
if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation":
op_type = op_type[:-10]
if op_type in opNames:
op_stat4[op_type] += 1
elif op[0] in opNames:
op_stat4[op[0]] += 1
self.assertTrue(block.identifier >= self.start)
self.assertTrue(block.identifier <= self.stop)
self.assertEqual(op_stat["transfer"], op_stat4["transfer"])
self.assertEqual(op_stat["vote"], op_stat4["vote"])
def test_stream_threading(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames, start=self.start, stop=self.stop):
ops_stream.append(op)
self.assertTrue(len(ops_stream) > 0)
op_stat = b.ops_statistics(start=self.start, stop=self.stop)
ops_blocks = []
for op in b.blocks(start=self.start,
stop=self.stop,
threading=True,
thread_num=8):
ops_blocks.append(op)
op_stat4 = {"transfer": 0, "vote": 0}
self.assertTrue(len(ops_blocks) > 0)
for block in ops_blocks:
for tran in block["transactions"]:
for op in tran['operations']:
if op[0] in opNames:
op_stat4[op[0]] += 1
self.assertTrue(block.identifier >= self.start)
self.assertTrue(block.identifier <= self.stop)
self.assertEqual(op_stat["transfer"], op_stat4["transfer"])
self.assertEqual(op_stat["vote"], op_stat4["vote"])
class Steem_node():
def __init__(self):
self.stm = Steem()
self.blockchain = Blockchain(steem_instance=self.stm, mode="head")
self.mailserver = Gmail(60)
def run(self):
# Fetch full blocks, then inspect each operation.
for block in self.blockchain.blocks():
block_num = block.block_num
created_at = block.time()
print(f'Block: {block_num}')
for op in block.operations:
# Extract op data
if op['type'] == 'transfer_operation':
type = 'transfer'
subject = 'New transfer'
to = op['value']['to']
FROM = op['value']['from']
amount = op['value']['amount']
memo = op['value']['memo']
# Check if account matches, sent mail on True
if to == 'steempytutorials':
print('Incoming transfer')
message = (
f'{created_at}\nBlock: {block_num}\nFrom: {FROM}' +
f'\nTo: {to}\nAmount: {amount}\nMemo: {memo}\n')
try:
self.mailserver.send_email(subject, message, type)
print('Mail sent')
except Exception as e:
print('Failed to sent mail', e)
def test_signing_appbase(self):
b = Blockchain(steem_instance=self.bts)
st = None
for block in b.blocks(start=25304468, stop=25304468):
for trx in block.transactions:
st = Signed_Transaction(trx.copy())
self.assertTrue(st is not None)
def find_steem_tx(self, tx_data: dict, last_blocks=15) -> dict:
"""
Used internally to get the transaction ID after a Steem transaction has been broadcasted.
See :py:meth:`.send_token` and :py:meth:`.issue_token` for how this is used.
:param dict tx_data: Transaction data returned by a beem broadcast operation, must include ``signatures``
:param int last_blocks: Amount of previous blocks to search for the transaction
:return dict: Transaction data from the blockchain, see below.
**Return format:**
.. code-block:: js
{
transaction_id, ref_block_num, ref_block_prefix, expiration,
operations, extensions, signatures, block_num, transaction_num
}
:return None: If the transaction wasn't found, ``None`` will be returned.
"""
from beem.blockchain import Blockchain
# Code taken/based from @holgern/beem blockchain.py
chain = Blockchain(steem_instance=self.steem, mode='head')
current_num = chain.get_current_block_num()
for block in chain.blocks(start=current_num - last_blocks, stop=current_num + 5):
for tx in block.transactions:
if sorted(tx["signatures"]) == sorted(tx_data["signatures"]):
return tx
return None
def test_block_threading(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
blocks_no_threading = []
for block in b.blocks(start=self.start,
stop=self.stop,
threading=False,
thread_num=8):
blocks_no_threading.append(block)
for n in range(5):
blocks = []
for block in b.blocks(start=self.start,
stop=self.stop,
threading=True,
thread_num=8):
blocks.append(block)
for i in range(min(len(blocks), len(blocks_no_threading))):
self.assertEqual(blocks[i]["block_id"],
blocks_no_threading[i]["block_id"])
self.assertEqual(len(blocks_no_threading), len(blocks))
def hour_active(chain):
nodelist = NodeList()
nodelist.update_nodes()
if chain=='steem':
s = Steem(node=nodelist.get_steem_nodes())
else:
s = Steem(node=nodelist.get_hive_nodes())
b=Blockchain(s)
bl_=[]
bl_num=int(b.get_current_block_num())
bl_num_=bl_num-1250
bl=b.blocks(bl_num_,bl_num)
for i in bl:
bl_.append(i['transactions'])
x=('follower','account','voter','from','author')
account=[]
new=[]
for i in x:
acc=re.findall('"'+i+'":"(.+?)"',str(bl_))
for l in acc:
if l not in account:
account.append(l)
if chain=='steem':
file=open('active_acc_steem.txt','r')
old=file.readlines()
file.close()
file=open('active_acc_steem.txt','a')
for i in account:
if i+'\n' not in old:
file.write(str(i)+'\n')
new.append(i)
file.close()
else:
file=open('active_acc_hive.txt','r')
old=file.readlines()
file.close()
file=open('active_acc_hive.txt','a')
for i in account:
if i+'\n' not in old:
file.write(str(i)+'\n')
new.append(i)
file.close()
print ('one hour accounts '+chain,len(account))
print ('one hour brand new '+chain,len(new))
print ('total ACTIVE accounts '+chain,len(old))
async def main():
blocks = []
hive = Hive(HIVE_NODES)
chain = Blockchain(blockchain_instance=hive)
print(f"\n [{datetime.utcnow()!s}] Loading last {NUM_BLOCKS} blocks using beem ... \n\n")
start_time = time.time_ns()
current_num = chain.get_current_block_num()
for block in chain.blocks(start=current_num - NUM_BLOCKS, stop=current_num):
blocks.append(block)
end_time = time.time_ns()
print(f"\n [{datetime.utcnow()!s}] Total blocks:", len(blocks), "\n")
start_time, end_time = Decimal(start_time), Decimal(end_time)
start_secs = start_time / SECS_NS
end_secs = end_time / SECS_NS
print("Start Time:", dec_round(start_secs, 4), "seconds")
print("End Time:", dec_round(end_secs, 4), "seconds\n")
print("Total Time:", dec_round(end_secs - start_secs, 4), "seconds\n")
def find_steem_tx(self, tx_data, last_blocks=15) -> Optional[dict]:
"""
Used internally to get the transaction ID after a transaction has been broadcasted
:param dict tx_data: Transaction data returned by a beem broadcast operation, must include 'signatures'
:param int last_blocks: Amount of previous blocks to search for the transaction
:return dict: Transaction data from the blockchain {transaction_id, ref_block_num, ref_block_prefix,
expiration, operations, extensions, signatures, block_num, transaction_num}
:return None: If the transaction wasn't found, None will be returned.
"""
# Code taken/based from @holgern/beem blockchain.py
chain = Blockchain(steem_instance=self.rpc, mode='head')
current_num = chain.get_current_block_num()
for block in chain.blocks(start=current_num - last_blocks, stop=current_num + 5):
for tx in block.transactions:
if sorted(tx["signatures"]) == sorted(tx_data["signatures"]):
return tx
return None
def benchmark_node(node, how_many_minutes=10, how_many_seconds=30):
block_count = 0
history_count = 0
access_time = 0
follow_time = 0
blockchain_version = u'0.0.0'
successful = True
error_msg = None
start_total = timer()
max_batch_size = None
threading = False
thread_num = 16
authorpermvoter = u"@gtg/steem-pressure-4-need-for-speed|gandalf"
[author, permlink, voter] = resolve_authorpermvoter(authorpermvoter)
authorperm = construct_authorperm(author, permlink)
last_block_id = 19273700
try:
stm = Steem(node=node, num_retries=3, num_retries_call=3, timeout=30)
blockchain = Blockchain(steem_instance=stm)
blockchain_version = stm.get_blockchain_version()
last_block = Block(last_block_id, steem_instance=stm)
stopTime = last_block.time() + timedelta(seconds=how_many_minutes * 60)
total_transaction = 0
start = timer()
for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num):
block_no = entry.identifier
block_count += 1
if "block" in entry:
trxs = entry["block"]["transactions"]
else:
trxs = entry["transactions"]
for tx in trxs:
for op in tx["operations"]:
total_transaction += 1
if "block" in entry:
block_time = parse_time(entry["block"]["timestamp"])
else:
block_time = parse_time(entry["timestamp"])
if block_time > stopTime:
last_block_id = block_no
break
if timer() - start > how_many_seconds or quit_thread:
break
except NumRetriesReached:
error_msg = 'NumRetriesReached'
block_count = -1
except KeyboardInterrupt:
error_msg = 'KeyboardInterrupt'
# quit = True
except Exception as e:
error_msg = str(e)
block_count = -1
try:
stm = Steem(node=node, num_retries=3, num_retries_call=3, timeout=30)
account = Account("gtg", steem_instance=stm)
blockchain_version = stm.get_blockchain_version()
start = timer()
for acc_op in account.history_reverse(batch_size=100):
history_count += 1
if timer() - start > how_many_seconds or quit_thread:
break
except NumRetriesReached:
error_msg = 'NumRetriesReached'
history_count = -1
successful = False
except KeyboardInterrupt:
error_msg = 'KeyboardInterrupt'
history_count = -1
successful = False
# quit = True
except Exception as e:
error_msg = str(e)
history_count = -1
successful = False
try:
stm = Steem(node=node, num_retries=3, num_retries_call=3, timeout=30)
account = Account("gtg", steem_instance=stm)
blockchain_version = stm.get_blockchain_version()
start = timer()
Vote(authorpermvoter, steem_instance=stm)
stop = timer()
vote_time = stop - start
start = timer()
Comment(authorperm, steem_instance=stm)
stop = timer()
comment_time = stop - start
start = timer()
Account(author, steem_instance=stm)
stop = timer()
account_time = stop - start
start = timer()
account.get_followers()
stop = timer()
follow_time = stop - start
access_time = (vote_time + comment_time + account_time + follow_time) / 4.0
except NumRetriesReached:
error_msg = 'NumRetriesReached'
access_time = -1
except KeyboardInterrupt:
error_msg = 'KeyboardInterrupt'
# quit = True
except Exception as e:
error_msg = str(e)
access_time = -1
return {'successful': successful, 'node': node, 'error': error_msg,
'total_duration': timer() - start_total, 'block_count': block_count,
'history_count': history_count, 'access_time': access_time, 'follow_time': follow_time,
'version': blockchain_version}
year = int(sys.argv[1])
month = int(sys.argv[2])
day = int(sys.argv[3])
nl = NodeList()
s = Steem(node=nl.get_nodes(appbase=True, normal=False))
b = Blockchain(steem_instance=s)
start_date = datetime(year, month, day)
end_date = start_date + timedelta(days=1)
start_block = b.get_estimated_block_num(start_date)
end_block = b.get_estimated_block_num(end_date) - 1
m = MongoStorage(db_name="steem",
host='172.18.0.3',
port=27017,
user='',
passwd='')
m.ensure_indexes()
for block in b.blocks(start=start_block,
stop=end_block,
max_batch_size=BATCH_SIZE):
sys.stdout.write("%s - %s - %s\r" %
(start_block, block.block_num, end_block))
for trx in block.transactions:
if "'vote_operation'" not in str(trx):
continue
m.Transactions.insert(trx)
def test_stream(self):
bts = self.bts
start = self.start
stop = self.stop
b = Blockchain(steem_instance=bts)
ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames, start=start, stop=stop):
ops_stream.append(op)
self.assertTrue(len(ops_stream) > 0)
ops_raw_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames,
raw_ops=True,
start=start,
stop=stop):
ops_raw_stream.append(op)
self.assertTrue(len(ops_raw_stream) > 0)
only_ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames,
start=start,
stop=stop,
only_ops=True):
only_ops_stream.append(op)
self.assertTrue(len(only_ops_stream) > 0)
only_ops_raw_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames,
raw_ops=True,
start=start,
stop=stop,
only_ops=True):
only_ops_raw_stream.append(op)
self.assertTrue(len(only_ops_raw_stream) > 0)
op_stat = b.ops_statistics(start=start, stop=stop)
op_stat2 = {"transfer": 0, "vote": 0}
for op in ops_stream:
self.assertIn(op["type"], opNames)
op_stat2[op["type"]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat2["transfer"])
self.assertEqual(op_stat["vote"], op_stat2["vote"])
op_stat3 = {"transfer": 0, "vote": 0}
for op in ops_raw_stream:
self.assertIn(op["op"][0], opNames)
op_stat3[op["op"][0]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat3["transfer"])
self.assertEqual(op_stat["vote"], op_stat3["vote"])
op_stat5 = {"transfer": 0, "vote": 0}
for op in only_ops_stream:
self.assertIn(op["type"], opNames)
op_stat5[op["type"]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat5["transfer"])
self.assertEqual(op_stat["vote"], op_stat5["vote"])
op_stat6 = {"transfer": 0, "vote": 0}
for op in only_ops_raw_stream:
self.assertIn(op["op"][0], opNames)
op_stat6[op["op"][0]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat6["transfer"])
self.assertEqual(op_stat["vote"], op_stat6["vote"])
ops_blocks = []
for op in b.blocks(start=start, stop=stop):
ops_blocks.append(op)
op_stat4 = {"transfer": 0, "vote": 0}
self.assertTrue(len(ops_blocks) > 0)
for block in ops_blocks:
for tran in block["transactions"]:
for op in tran['operations']:
if isinstance(op, list) and op[0] in opNames:
op_stat4[op[0]] += 1
elif isinstance(op, dict):
op_type = op["type"]
if len(op_type) > 10 and op_type[len(op_type) -
10:] == "_operation":
op_type = op_type[:-10]
if op_type in opNames:
op_stat4[op_type] += 1
self.assertTrue(block.identifier >= start)
self.assertTrue(block.identifier <= stop)
self.assertEqual(op_stat["transfer"], op_stat4["transfer"])
self.assertEqual(op_stat["vote"], op_stat4["vote"])
ops_blocks = []
for op in b.blocks():
ops_blocks.append(op)
break
self.assertTrue(len(ops_blocks) == 1)
class LatestPostManager():
def __init__(self):
self.mysql_con = config.get_connection()
if self.mysql_con is None:
print(
"[INFO] Can't start Latest Post Manager because of an mysql database error!"
)
return
self.mysql_cursor = self.mysql_con.cursor()
self.query = "INSERT INTO latest_posts (author, permlink, category, timestamp) VALUES (%s, %s, %s, %s);"
self.chain = Blockchain(
blockchain_instance=Hive()) #node=conf.HIVE_NODES[5]
self.run_thread = Thread(target=self.run)
self.run_thread.name = 'Get & Categorize Posts'
self.run_thread.daemon = True
self.run_thread.start()
def enter_posts_by_block(self, block):
for op in block.operations:
if op['type'] == 'comment_operation':
action = op['value']
if action['parent_author'] == '':
# found post --> Categorize
_input = network.WordEmbedding.vectorize_text(
config.statics.Word2Vec,
html=action['body'],
text=action['title'] + ". ")
if _input is None:
# to short or error
continue
# Categorize
_output = config.statics.TextCNN(_input).cpu()
# Enter in Mysql
str_arr = ' '.join(map(str, _output.data[0].tolist()))
result = database.commit_query(
self.query,
(action['author'], action['permlink'], str_arr,
block["timestamp"].strftime("%d.%m.%YT%H:%M:%S")))
if result == -1:
print("[WARNING] Can't enter post in database!")
time.sleep(5)
def clean_up(self):
# TODO: Implement sorting order by timestamp
query = "SELECT timestamp FROM latest_posts;"
for item in database.read_query(query, None):
timestamp = datetime.strptime(item[0], "%d.%m.%YT%H:%M:%S")
if timestamp < (datetime.utcnow() - timedelta(days=5)):
result = database.commit_query(
"DELETE FROM latest_posts WHERE timestamp=%s;",
(item[0], ))
def run(self):
current_num = self.chain.get_current_block_num() - int(
60 * 60 * 24 * 3 / 3
) # Get all posts from the last 3 days because it takes a long time to get all and when it finished, the clean_up begins
while True:
if current_num < self.chain.get_current_block_num():
# if block is available
try:
for block in self.chain.blocks(start=current_num,
stop=current_num):
self.enter_posts_by_block(block)
except:
pass
current_num += 1
if current_num % 100 == 0:
self.clean_up()
else:
time.sleep(0.5)
else:
# wait until new block is created
# Using time for cleanup
self.clean_up()
def benchmark_node_blocks(node,
num_retries=10,
num_retries_call=10,
timeout=60,
how_many_seconds=30):
block_count = 0
sucessfull = False
error_msg = None
start_total = timer()
start_time = timer()
max_batch_size = None
threading = False
thread_num = 16
last_block_id = 19273700
try:
stm = Steem(node=node,
num_retries=num_retries,
num_retries_call=num_retries_call,
timeout=timeout)
blockchain = Blockchain(steem_instance=stm)
last_block_id = int(blockchain.get_current_block_num() * 0.75)
last_block = Block(last_block_id, steem_instance=stm)
total_transaction = 0
start_time = timer()
for entry in blockchain.blocks(start=last_block_id,
max_batch_size=max_batch_size,
threading=threading,
thread_num=thread_num):
block_no = entry.identifier
block_count += 1
if not sucessfull:
sucessfull = True
if "block" in entry:
trxs = entry["block"]["transactions"]
else:
trxs = entry["transactions"]
for tx in trxs:
for op in tx["operations"]:
total_transaction += 1
if "block" in entry:
block_time = (entry["block"]["timestamp"])
else:
block_time = (entry["timestamp"])
if timer() - start_time > how_many_seconds or quit_thread:
break
except NumRetriesReached:
error_msg = 'NumRetriesReached'
sucessfull = False
except KeyboardInterrupt:
error_msg = 'KeyboardInterrupt'
# quit = True
except Exception as e:
error_msg = str(e)
sucessfull = False
total_duration = float("{0:.2f}".format(timer() - start_time))
return {
'sucessfull': sucessfull,
'node': node,
'error': error_msg,
'total_duration': total_duration,
'count': block_count,
'access_time': None
}
year = 2019
month = int(sys.argv[1])
day = int(sys.argv[2])
start_date = datetime(year, month, day)
stop_date = start_date + timedelta(days=1)
DW_IDs = ['dw-heist', 'drugwars', 'dw-unit', 'dw-upgrade', 'dw-char']
stm = None #Steem(node=['https://api.steemit.com'])
b = Blockchain(steem_instance=stm)
start_block = b.get_estimated_block_num(start_date)
stop_block = b.get_estimated_block_num(stop_date)
transactions = []
for blk in b.blocks(start=start_block, stop=stop_block, threading=True):
# max_batch_size=50):
sys.stdout.write("%s\r" % (blk['timestamp']))
for trx in blk.transactions:
keep = False
for op in trx['operations']:
if op['type'] == 'custom_json_operation' and \
op['value']['id'] in DW_IDs:
keep = True
if keep:
transactions.append(trx)
s = shelve.open("transactions_%04d%02d%02d.shelf" % (year, month, day))
s['transactions'] = transactions
s.close()
def get_transactions(start_block_num, log_data):
config_data = read_config()
databaseConnector = config_data["databaseConnector"]
custom_json_id = config_data["custom_json_id"]
transfer_id = config_data["transfer_id"]
current_block = start_block_num - 1
nodes = NodeList()
# weights = {'block': 1, 'history': 0.5, 'apicall': 0.5, 'config': 0.5}
nodes.update_nodes()
stm = Steem(node=["https://api.steemit.com"])
print(stm)
b = Blockchain(mode="head", steem_instance=stm)
if "stop_block_num" in config_data and config_data["stop_block_num"] > 0:
stop_block_num = config_data["stop_block_num"]
else:
stop_block_num = b.get_current_block_num()
if stop_block_num - start_block_num > 1200:
stop_block_num = start_block_num + 1200
log_data["stop_block_num"] = stop_block_num
db = dataset.connect(databaseConnector)
table = db['status']
table.upsert({"id": 1, "tracker_stop_block_num": stop_block_num}, ["id"])
table = db["blocks"]
data = table.find_one(order_by='-block_num')
if data is None:
latest_block_num = start_block_num - 1
else:
latest_block_num = data["block_num"]
table = db["virtualops"]
for block in b.blocks(start=start_block_num, stop=stop_block_num, threading=False, thread_num=8):
if "transactions" in block:
trx = block["transactions"]
else:
trx = [block]
block_num = 0
trx_id = ""
_id = ""
timestamp = ""
for trx_nr in range(len(trx)):
if "operations" not in trx[trx_nr]:
continue
for event in trx[trx_nr]["operations"]:
if isinstance(event, list):
op_type, op = event
trx_id = block["transaction_ids"][trx_nr]
block_num = block.get("id")
timestamp = block.get("timestamp")
block_id = block.get("block_id")
previous = block.get("previous")
elif isinstance(event, dict) and "type" in event and "value" in event:
op_type = event["type"]
if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation":
op_type = op_type[:-10]
op = event["value"]
trx_id = block["transaction_ids"][trx_nr]
block_num = block.get("id")
timestamp = block.get("timestamp")
block_id = block.get("block_id")
previous = block.get("previous")
elif "op" in event and isinstance(event["op"], dict) and "type" in event["op"] and "value" in event["op"]:
op_type = event["op"]["type"]
if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation":
op_type = op_type[:-10]
op = event["op"]["value"]
trx_id = event.get("trx_id")
block_num = event.get("block")
timestamp = event.get("timestamp")
block_id = None
previous = None
else:
op_type, op = event["op"]
trx_id = event.get("trx_id")
block_num = event.get("block")
timestamp = event.get("timestamp")
block_id = None
previous = None
# print(op)
log_data = print_block_log(start_block_num, log_data, block_num, timestamp)
current_block = block_num
date = timestamp.replace(tzinfo=None)
if block_num > latest_block_num:
table = db["blocks"]
table.insert({"block_num": block_num, "timestamp": date, "block_id": block_id, "previous": previous})
latest_block_num = block_num
if op_type == "transfer":
if op["to"] != "nextcolony":
continue
user = op["from"]
memo = op["memo"]
at_symbol_pos = memo.find('@')
if at_symbol_pos < 0:
continue
if memo[:at_symbol_pos] != transfer_id:
continue
amount = Amount(op["amount"], steem_instance=stm)
log_data["new_transfers"] += 1
table = db['transfers']
if table.find_one(trx=trx_id) == None:
print ("Writing the transaction into the DB")
data = dict(trx=trx_id, user=user, block_num=int(block_num), amount=str(amount), memo=memo, tr_status=0, date=date)
table.insert(data)
table = db['status']
table.upsert({"id": 1, "latest_block_num": int(block_num)}, ["id"])
elif op_type == "custom_json":
if op['id'] != custom_json_id:
continue
date = timestamp.replace(tzinfo=None)
try:
json_data = json.loads(op['json'])
except:
print("Skip json: %s" % str(op['json']))
continue
if isinstance(json_data, list) and len(json_data) == 1:
json_data = json_data[0]
if not isinstance(json_data, dict):
print(json_data)
print("not a valid json")
continue
if 'command' not in json_data:
print (json_data)
print("wrong cusom_json structure")
continue
if len(json_data['command']) == 0:
print(json_data)
print("command is empty")
continue
if len(op['required_posting_auths']) > 0:
user = op['required_posting_auths'][0]
elif len(op['required_auths']) > 0:
user = op['required_auths'][0]
else:
print("Cannot parse transaction, as user could not be determined!")
continue
table = db['transactions']
if table.find_one(trx=trx_id) != None:
print("Skip trx_id %s at %s" % (trx_id, date))
continue
log_data["new_custom_json"] += 1
command = json_data['command']
if "type" not in json_data:
continue
if json_data["type"] is None:
continue
json_type = json_data['type']
print("%s: %d:%s - @%s type: %s, command: %s" % (str(date), block_num, trx_id, user, str(json_type), str(command)))
if not isinstance(command, dict):
print("Skip trx_id %s at %s" % (trx_id, date))
continue
if "tr_var1" in command:
tr_var1 = unidecode(str(command['tr_var1']))
else:
tr_var1 = 0
if "tr_var2" in command:
tr_var2 = unidecode(str(command['tr_var2']))
else:
tr_var2 = 0
if "tr_var3" in command:
tr_var3 = unidecode(str(command['tr_var3']))
else:
tr_var3 = 0
if "tr_var4" in command:
tr_var4 = unidecode(str(command['tr_var4']))
else:
tr_var4 = 0
if "tr_var5" in command:
tr_var5 = unidecode(str(command['tr_var5']))
else:
tr_var5 = 0
if "tr_var6" in command:
tr_var6 = unidecode(str(command['tr_var6']))
else:
tr_var6 = 0
if "tr_var7" in command:
tr_var7 = unidecode(str(command['tr_var7']))
else:
tr_var7 = 0
if "tr_var8" in command:
tr_var8 = unidecode(str(command['tr_var8']))
else:
tr_var8 = 0
table = db['transactions']
data = dict(trx=trx_id, user=user, tr_type=json_type, block_num=int(block_num), tr_var1=str(tr_var1),
tr_var2=str(tr_var2), tr_var3=tr_var3, tr_var4=tr_var4, tr_var5=tr_var5,
tr_var6=tr_var6, tr_var7=tr_var7, tr_var8=tr_var8, tr_status=0 ,date=date)
table.insert(data)
table = db['status']
table.upsert({"id": 1, "latest_block_num": int(block_num)}, ["id"])
return current_block + 1, log_data
last_block_id = 19273700
last_block = Block(last_block_id, steem_instance=stm)
startTime = datetime.now()
stopTime = last_block.time() + timedelta(seconds=how_many_minutes *
60)
ltime = time.time()
cnt = 0
total_transaction = 0
start_time = time.time()
last_node = blockchain.steem.rpc.url
for entry in blockchain.blocks(start=last_block_id,
max_batch_size=max_batch_size,
threading=threading,
thread_num=thread_num):
block_no = entry.identifier
if "block" in entry:
trxs = entry["block"]["transactions"]
else:
trxs = entry["transactions"]
for tx in trxs:
for op in tx["operations"]:
total_transaction += 1
if "block" in entry:
block_time = parse_time(entry["block"]["timestamp"])
else:
block_time = parse_time(entry["timestamp"])
def main(args=None):
args = parse_args(args)
blockchain = args.blockchain
nodelist = NodeList()
nodelist.update_nodes(weights={"block": 1})
if blockchain == "hive" or blockchain is None:
max_batch_size = 50
threading = False
thread_num = 16
block_debug = 1000
nodes = nodelist.get_hive_nodes()
blk_inst = Hive(node=nodes,
num_retries=3,
num_retries_call=3,
timeout=30)
elif blockchain == "blurt":
max_batch_size = None
threading = False
thread_num = 8
block_debug = 20
nodes = [
"https://api.blurt.blog", "https://rpc.blurtworld.com",
"https://rpc.blurtworld.com"
]
blk_inst = Blurt(node=nodes,
num_retries=3,
num_retries_call=3,
timeout=30)
elif blockchain == "steem":
max_batch_size = 50
threading = False
thread_num = 16
block_debug = 1000
nodes = nodelist.get_steem_nodes()
blk_inst = Steem(node=nodes,
num_retries=3,
num_retries_call=3,
timeout=30)
else:
raise Exception("Wrong parameter, can be hive, blurt or steem")
print(blk_inst)
block_count = 0
total_ops = 0
total_virtual_ops = 0
total_trx = 0
blocksperday = 20 * 60 * 24
blockchain = Blockchain(blockchain_instance=blk_inst, )
current_block_num = blockchain.get_current_block_num()
last_block_id = current_block_num - blocksperday
last_block = Block(last_block_id, blockchain_instance=blk_inst)
stopTime = last_block.time() + timedelta(seconds=60 * 60 * 24)
start = timer()
for entry in blockchain.blocks(start=last_block_id,
max_batch_size=max_batch_size,
threading=threading,
thread_num=thread_num):
if "block" in entry:
block_time = parse_time(entry["block"]["timestamp"])
else:
block_time = entry["timestamp"]
if block_time > stopTime:
break
block_count += 1
if "block" in entry:
trxs = entry["block"]["transactions"]
else:
trxs = entry["transactions"]
for tx in trxs:
total_trx += 1
for op in tx["operations"]:
total_ops += 1
ops_per_day = total_ops / block_count * blocksperday
if block_count % (block_debug) == 0:
print("%d blocks remaining... estimated ops per day: %.1f" %
(blocksperday - block_count, ops_per_day))
duration = timer() - start
stopTime = last_block.time() + timedelta(seconds=60 * 60 * 24)
start = timer()
for entry in blockchain.blocks(start=last_block_id,
max_batch_size=max_batch_size,
threading=threading,
thread_num=thread_num,
only_virtual_ops=True):
block_time = entry["timestamp"]
if block_time > stopTime:
break
for tx in entry["operations"]:
for op in tx["op"]:
total_virtual_ops += 1
duration = timer() - start
print("Received %.2f blocks/s." % (block_count / duration))
print("Bocks: %d, duration %.3f s" % (block_count, duration))
print("Operations per day: %d" % total_ops)
print("Trx per day: %d" % total_trx)
print("Virtual Operations per day: %d" % total_virtual_ops)
def main(args=None):
args = parse_args(args)
blockchain = args.blockchain
nodelist = NodeList()
nodelist.update_nodes(weights={"block": 1})
if blockchain == "hive" or blockchain is None:
max_batch_size = 50
threading = False
thread_num = 16
block_debug = 1000
nodes = nodelist.get_hive_nodes()
blk_inst = Hive(node=nodes, num_retries=3, num_retries_call=3, timeout=30)
elif blockchain == "blurt":
max_batch_size = None
threading = False
thread_num = 8
block_debug = 20
nodes = ["https://rpc.blurt.buzz/", "https://api.blurt.blog", "https://rpc.blurtworld.com", "https://rpc.blurtworld.com"]
blk_inst = Blurt(node=nodes, num_retries=3, num_retries_call=3, timeout=30)
elif blockchain == "steem":
max_batch_size = 50
threading = False
thread_num = 16
block_debug = 1000
nodes = nodelist.get_steem_nodes()
blk_inst = Steem(node=nodes, num_retries=3, num_retries_call=3, timeout=30)
else:
raise Exception("Wrong parameter, can be hive, blurt or steem")
print(blk_inst)
block_count = 0
total_ops = 0
total_trx = 0
duration_s = 60 * 60 * 1
blocksperday = int(duration_s / 3)
blockchain = Blockchain(blockchain_instance=blk_inst, )
current_block_num = blockchain.get_current_block_num()
last_block_id = current_block_num - blocksperday
last_block = Block(last_block_id, blockchain_instance=blk_inst)
stopTime = last_block.time() + timedelta(seconds=duration_s)
start = timer()
op_stats = {}
for entry in blockchain.blocks(start=last_block_id, max_batch_size=max_batch_size, threading=threading, thread_num=thread_num):
if "block" in entry:
block_time = parse_time(entry["block"]["timestamp"])
else:
block_time = entry["timestamp"]
if block_time > stopTime:
break
block_count += 1
if "block" in entry:
trxs = entry["block"]["transactions"]
else:
trxs = entry["transactions"]
for tx in trxs:
total_trx += 1
for op in tx["operations"]:
if "_operation" in op["type"]:
op_type = op["type"][:-10]
else:
op_type = op["type"]
if op_type in op_stats:
op_stats[op_type] += 1
else:
op_stats[op_type] = 1
total_ops += 1
ops_per_day = total_ops / block_count * blocksperday
if block_count % (block_debug) == 0:
print("%d blocks remaining... estimated ops per day: %.1f" % (blocksperday - block_count, ops_per_day))
duration = timer() - start
t = PrettyTable(["Type", "Count", "percentage"])
t.align = "l"
op_list = []
for o in op_stats:
op_list.append({"type": o, "n": op_stats[o], "perc": op_stats[o] / total_ops * 100})
op_list_sorted = sorted(op_list, key=lambda x: x['n'], reverse=True)
for op in op_list_sorted:
t.add_row([op["type"], op["n"], "%.2f %%" % op["perc"]])
print(t)
import sys
import shelve
from beem.blockchain import Blockchain
from datetime import datetime, timedelta
start_date = datetime(2019, 3, 5)
stop_date = start_date + timedelta(days=1)
DW_IDs = ['dw-heist', 'drugwars', 'dw-unit', 'dw-upgrade', 'dw-char']
b = Blockchain()
start_block = b.get_estimated_block_num(start_date)
stop_block = b.get_estimated_block_num(stop_date)
transactions = []
for blk in b.blocks(start=start_block, stop=stop_block, max_batch_size=50):
sys.stdout.write("%s\r" % (blk['timestamp']))
for trx in blk.transactions:
keep = False
for op in trx['operations']:
if op['type'] == 'custom_json_operation' and \
op['value']['id'] in DW_IDs:
keep = True
if keep:
transactions.append(trx)
s = shelve.open("transactions.shelf")
s['transactions'] = transactions
s.close()
from beem.blockchain import Blockchain
import time
nodes = ["wss://steemd.pevo.science"]
threading = True
thread_num = 16
stm = beem.steem.Steem(nodes)
b = Blockchain(steem_instance=stm)
starttime = time.time()
dvcount = 0
firstblock = 18581583
while True:
lastblock = firstblock + 100
if lastblock > 18610152:
lastblock = 18610152
blocks = b.blocks(firstblock,
lastblock,
threading=threading,
thread_num=thread_num)
for block in blocks:
for t in block["transactions"]:
for o in t["operations"]:
if o[0] == "vote" and o[1]["weight"] < 0:
dvcount = dvcount + 1
if dvcount % 250 == 0:
print("Processed", dvcount)
firstblock = lastblock
if firstblock >= 18610152:
break
endtime = time.time()
print("dvcount = ", dvcount)
print("Processing January first of 2018 took", endtime - starttime, "seconds")
