def one_block_check(block_num):
logger.info('check block number: {}'.format(block_num))
try:
block = gph.rpc.get_block(block_num)
#witness_id = block['witness']
block_witness = gph.rpc.get_object(
gph.rpc.get_object(
block['witness'])['witness_account'])['name']
except Exception as e:
logger.error('get_object exception. block {}, error {}'.format(
block_num, repr(e)))
return
block_time = block['timestamp']
transactions = block["transactions"]
witness_sign = block['witness_signature']
trx_total = 0
ops_total = 0
transactions_id = []
if transactions:
trx_total = len(transactions)
for trx in transactions:
transactions_id.append(trx[0])
ops_total += len(trx[1]["operations"])
block_data = {
"block_num": block_num,
"time": block_time,
"witness": block_witness,
"witness_sign": witness_sign,
"transactions_total": trx_total,
"transactions_id": transactions_id,
"operations_total": ops_total
}
block_info_deque_lock.acquire()
block_info_q.append(block_data)
block_info_deque_lock.release()
def analysis_block():
gph = Graphene(node=nodeaddress)
from PythonMiddleware.instance import set_shared_graphene_instance
set_shared_graphene_instance(gph)
while 1:
if pending_block_num_q:
try:
pending_block_num_deque_lock.acquire()
block_num = pending_block_num_q.popleft()
pending_block_num_deque_lock.release()
logger.debug('pop block number: {}'.format(block_num))
try:
block_info = gph.rpc.get_block(block_num)
time = block_info["timestamp"]
transactions = block_info["transactions"]
operations_list = parse_operations(gph, block_num, time,
transactions)
#logger.debug('block: {}, trx_list: {}'.format(block_num, operations_list))
except Exception as e:
logger.error(
'parse block exception. block {}, error {}'.format(
block_num, repr(e)))
if operations_list:
op_deque_lock.acquire()
operations_q.append(operations_list)
op_deque_lock.release()
except Exception as e:
logger.error("pending_block_num_q: {}, except: '{}'".format(
pending_block_num_q, repr(e)))
sleep(0.7)
def check_block(args):
def one_block_check(block_num):
logger.info('recv block number: {}'.format(block_num))
try:
block = gph.rpc.get_block(block_num)
#witness_id = block['witness']
block_witness = gph.rpc.get_object(
gph.rpc.get_object(
block['witness'])['witness_account'])['name']
except Exception as e:
logger.error('get_object exception. block {}, error {}'.format(
block_num, repr(e)))
block_time = block['timestamp']
transactions = block["transactions"]
witness_sign = block['witness_signature']
trx_total = 0
ops_total = 0
transactions_id = []
if transactions:
trx_total = len(transactions)
for trx in transactions:
transactions_id.append(trx[0])
ops_total += len(trx[1]["operations"])
block_data = {
"block_num": block_num,
"time": block_time,
"witness": block_witness,
"witness_sign": witness_sign,
"transactions_total": trx_total,
"transactions_id": transactions_id,
"operations_total": ops_total
}
block_info_deque_lock.acquire()
block_info_q.append(block_data)
block_info_deque_lock.release()
start = args[0]
end = args[1]
gph = Graphene(node=nodeaddress)
info = gph.info()
last_block_num = info['head_block_number']
#logger.info('last_block_num: {}, block start: {}, end: {}, info: {}'.format(last_block_num, start, end, info))
logger.info('last_block_num: {}, block start: {}, end: {}'.format(
last_block_num, start, end))
if start > last_block_num:
logger.error("start:{} < end:{}".format(start, end))
return
if end > last_block_num:
end = last_block_num
conn = pymongo.MongoClient(mongodb_params['host'], mongodb_params['port'])
conn_db = conn[mongodb_params['db_name']]
for index in range(start, end + 1):
result = conn_db.block.find({'block_num': index})
if result.count() == 0:
logger.info('check block number: {}'.format(index))
one_block_check(index)
else:
logger.info('block({}) already exists in mongodb'.format(index))
sleep(0.1)
conn.close()
def on_block_callback(recv_block_id):
info = gph.info()
#logger.debug('info: {}'.format(info))
head_block_id = info['head_block_id']
block_num = info['head_block_number']
logger.info(
'recv_block_id: {}, head_block_id {}, head_block_num {}'.format(
recv_block_id, head_block_id, block_num))
if recv_block_id == head_block_id:
pending_block_num_deque_lock.acquire()
pending_block_num_q.append(block_num)
pending_block_num_deque_lock.release()
logger.info("pending deque >>>: {}".format(pending_block_num_q))
try:
block = gph.rpc.get_block(block_num)
#witness_id = block['witness']
# block_witness = gph.rpc.get_object(gph.rpc.get_object(block['witness'])['witness_account'])['name']
witness_obj = gph.rpc.get_object(block['witness'])
witness_account_obj = gph.rpc.get_object(
witness_obj['witness_account'])
except Exception as e:
logger.error('get_object exception. block {}, error {}'.format(
block_num, repr(e)))
block_time = block['timestamp']
transactions = block["transactions"]
witness_sign = block['witness_signature']
trx_total = 0
ops_total = 0
transactions_id = []
if transactions:
trx_total = len(transactions)
for trx in transactions:
transactions_id.append(trx[0])
ops_total += len(trx[1]["operations"])
block_data = {
"block_num": block_num,
"time": block_time,
"witness_id": witness_obj["id"],
"witness_account_id": witness_account_obj['id'],
"witness_account_name": witness_account_obj["name"],
"witness_sign": witness_sign,
"transactions_total": trx_total,
"transactions_id": transactions_id,
"operations_total": ops_total
}
block_info_deque_lock.acquire()
block_info_q.append(block_data)
block_info_deque_lock.release()
def data2db():
while 1:
if operations_q:
try:
op_deque_lock.acquire()
operations_list = operations_q.popleft()
op_deque_lock.release()
handle_operations(operations_list)
# status = handle_operations(operations_list)
# if not status:
# op_deque_lock.acquire()
# block_trx_ops = operations_q.appendleft(operations_list)
# op_deque_lock.release()
# logger.warn('consume status {}, trx list: {}'.format(status, operations_list))
except Exception as e:
logger.error("except: '{}'".format(repr(e)))
sleep(0.5)
def one_block_check(block_num):
logger.info('recv block number: {}'.format(block_num))
try:
block = gph.rpc.get_block(block_num)
if block is None:
logger.debug("block({}) is None. block num: {}".format(
block, block_num))
return
#witness_id = block['witness']
witness_obj = gph.rpc.get_object(block['witness'])
witness_account_obj = gph.rpc.get_object(
witness_obj['witness_account'])
except Exception as e:
logger.error('get_object exception. block {}, error {}'.format(
block_num, repr(e)))
return
block_time = block['timestamp']
transactions = block["transactions"]
witness_sign = block['witness_signature']
trx_total = 0
ops_total = 0
transactions_id = []
if transactions:
trx_total = len(transactions)
for trx in transactions:
transactions_id.append(trx[0])
ops_total += len(trx[1]["operations"])
block_data = {
"block_num": block_num,
"time": block_time,
"witness_id": witness_obj["id"],
"witness_account_id": witness_account_obj['id'],
"witness_account_name": witness_account_obj["name"],
"witness_sign": witness_sign,
"transactions_total": trx_total,
"transactions_id": transactions_id,
"operations_total": ops_total
}
block_info_deque_lock.acquire()
block_info_q.append(block_data)
block_info_deque_lock.release()
def block2db():
while 1:
if block_info_q:
try:
#global block_info_deque_lock
block_info_deque_lock.acquire()
block = block_info_q.popleft()
block_info_deque_lock.release()
#update mongodb
conn = pymongo.MongoClient(mongodb_params['host'],
mongodb_params['port'])
conn_db = conn[mongodb_params['db_name']]
try:
conn_db.block.insert_one({
'block_num':
block["block_num"],
'time':
block["time"],
'witness':
block["witness"],
'witness_sign':
block["witness_sign"],
'transactions_id':
str(block["transactions_id"]),
'transactions_total':
block["transactions_total"],
'operations_total':
block["operations_total"]
})
except Exception as e:
logger.error("block: {}, except: '{}'".format(
block["block_num"], repr(e)))
finally:
conn.close()
logger.info('block num: {} done.'.format(block["block_num"]))
except Exception as e:
logger.error("except: '{}'".format(repr(e)))
sleep(0.7)
def check_block():
def one_block_check(block_num):
logger.info('check block number: {}'.format(block_num))
try:
block = gph.rpc.get_block(block_num)
#witness_id = block['witness']
block_witness = gph.rpc.get_object(
gph.rpc.get_object(
block['witness'])['witness_account'])['name']
except Exception as e:
logger.error('get_object exception. block {}, error {}'.format(
block_num, repr(e)))
return
block_time = block['timestamp']
transactions = block["transactions"]
witness_sign = block['witness_signature']
trx_total = 0
ops_total = 0
transactions_id = []
if transactions:
trx_total = len(transactions)
for trx in transactions:
transactions_id.append(trx[0])
ops_total += len(trx[1]["operations"])
block_data = {
"block_num": block_num,
"time": block_time,
"witness": block_witness,
"witness_sign": witness_sign,
"transactions_total": trx_total,
"transactions_id": transactions_id,
"operations_total": ops_total
}
block_info_deque_lock.acquire()
block_info_q.append(block_data)
block_info_deque_lock.release()
try:
gph = Graphene(node=nodeaddress)
#info = gph.info()
#logger.info('info: {}'.format(info))
conn = pymongo.MongoClient(mongodb_params['host'],
mongodb_params['port'])
conn_db = conn[mongodb_params['db_name']]
sort_limit_result = conn_db.block.find().sort("block_num", -1).limit(1)
db_last_block_str = jsonb.dumps(sort_limit_result)
logger.info("db_last_block_str: {}".format(db_last_block_str))
db_last_block = json.loads(db_last_block_str)
if len(db_last_block) != 1:
logger.error(
"conn_db.block.find().sort('block_num', -1).limit(1) exception"
)
conn.close()
return
start_num = db_last_block[0]['block_num']
info = gph.info()
logger.info('info: {}'.format(info))
last_block_num = info['head_block_number']
increase = int(
(last_block_num - start_num) / 8) + 10 # 尽可能处理listen之前新增的区块
logger.info(">>> start: {}, end: {}, step: {}".format(
start_num, last_block_num, increase))
for index in range(start_num, last_block_num + increase):
result = conn_db.block.find({'block_num': index})
if result.count() != 0:
logger.info(
'block({}) already exists in mongo db'.format(index))
continue
one_block_check(index)
conn.close()
except Exception as e:
logger.error("except: '{}'".format(repr(e)))
# status = handle_operations(operations_list)
# if not status:
# op_deque_lock.acquire()
# block_trx_ops = op_d.appendleft(operations_list)
# op_deque_lock.release()
# logger.warn('consume status {}, trx list: {}'.format(status, operations_list))
except Exception as e:
logger.error("except: '{}'".format(repr(e)))
sleep(0.5)
if __name__ == '__main__':
logger.info('args: {}'.format(sys.argv))
if len(sys.argv) < 3:
logger.error(
'Usage: python3 check.py block_number_start, block_number_end')
sys.exit(1)
start = int(sys.argv[1])
end = int(sys.argv[2])
if start > end or start <= 0 or end <= 0:
logger.error(
'block_number_start: {} > block_number_end: {} or start <= 0 or end <= 0'
.format(start, end))
sys.exit(1)
args = [start, end]
init_gauges()
t1 = Thread(target=check_block, args=(args, ))
t1.start()
t2 = Thread(target=block2db)
t2.start()
t3 = Thread(target=analysis_block)
def block2db():
while 1:
if block_info_q:
try:
#global block_info_deque_lock
block_info_deque_lock.acquire()
block = block_info_q.popleft()
block_info_deque_lock.release()
# logger.debug(">>> block: {}".format(block))
#update mongodb
conn = pymongo.MongoClient(mongodb_params['host'],
mongodb_params['port'])
conn_db = conn[mongodb_params['db_name']]
try:
conn_db.block.insert_one({
'block_num':
block["block_num"],
'time':
block["time"],
'witness':
block["witness_account_name"],
'witness_id':
block["witness_id"],
'witness_account_id':
block["witness_account_id"],
# 'witness_sign': block["witness_sign"],
'witness_sign':
"",
'transactions_id':
str(block["transactions_id"]),
'transactions_total':
block["transactions_total"],
'operations_total':
block["operations_total"]
})
# witness generate block count
witness_blocks = {
'witness_id': block["witness_id"],
'witness_account_id': block["witness_account_id"],
'witness_account_name': block["witness_account_name"],
"last_block": str(block["block_num"]),
"last_block_time": block["time"]
}
witness_object = conn_db.witnesses.find_one(
{'witness_id': block["witness_id"]})
if (witness_object is not None):
if (int(block["block_num"]) > int(
witness_object["last_block"])):
witness_blocks['_id'] = witness_object['_id']
witness_blocks["total"] = str(
int(witness_object["total"]) + 1)
conn_db.witnesses.save(witness_blocks)
else:
logger.info("witness({}) block:{}, not exists.".format(
block["witness_id"], block["block_num"]))
witness_blocks["total"] = "1"
conn_db.witnesses.save(witness_blocks)
except Exception as e:
logger.error("block: {}, except: '{}'".format(
block["block_num"], repr(e)))
finally:
conn.close()
logger.info('block num: {} done.'.format(block["block_num"]))
except Exception as e:
logger.error("except: '{}'".format(repr(e)))
sleep(0.7)
