def main():
config, err = ConfigParser.Parse()
if err is not None:
logger.error("Failed to get hawq config: {}".format(err))
exit(-1)
transWriter = TransWriter(config)
start = datetime.datetime.now()
count = 1
try:
blockDb = plyvel.DB(config.get("blockDb"))
blockIndexDb = plyvel.DB(config.get("blockIndexDb"))
contract.initContractParser()
for i in range(config.get("start_num"), config.get("end_num")):
try:
if count % 1000 == 0:
end = datetime.datetime.now()
logger.info("已处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count,
(end - start).microseconds,
(end - start).microseconds / count,
))
transWriter.flush()
blockHashBytes = blockIndexDb.get(num2Bytes(i))
blockBytes = blockDb.get(blockHashBytes)
blockHash = bytes2HexStr(blockHashBytes)
block = Tron_pb2.Block()
block.ParseFromString(blockBytes)
appendData = {"block_num": i, "hash": blockHash}
bp = BlockParser()
ret = bp.Parse(transWriter, block, appendData)
if not ret:
logger.error("Failed to parse block num: {}".format(i))
break
count += 1
except Exception:
logger.error("Failed to parse block num: {}".format(i))
traceback.print_exc()
transWriter.write("error_block_num", [i])
break
# except Exception as e:
# logger.error("Failed to parse block num: {}".format(i))
# traceback.print_exc()
# raise e
# break
# end = datetime.datetime.now()
# logger.info(
# "共处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
# count - 1,
# (end - start).microseconds,
# (end - start).microseconds / (count - 1),
# )
# )
except Exception:
traceback.print_exc()
finally:
transWriter.flush()
transWriter.close()
end = datetime.datetime.now()
logger.info("共处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count - 1,
(end - start).microseconds,
(end - start).microseconds / (count - 1),
))
logger.info("处理 29617377 个区块，预计用时 {} 小时".format(
((29617377 / (count - 1)) *
(end - start).microseconds) / 1000000 / 3600))
logger.info("开始时间: {}".format(start.strftime("%Y-%m-%d %H:%M:%S")))
logger.info("结束时间: {}".format(end.strftime("%Y-%m-%d %H:%M:%S")))
def main():
total = (2750 - 2250) * 10**4
config, err = TransConfigParser.Parse()
if err is not None:
logger.error("Failed to get block config: {}".format(err))
exit(-1)
transWriter = TransWriter(config, tables)
start = datetime.datetime.now()
count = 1
try:
blockDb = plyvel.DB(config.get("blockDb"))
blockIndexDb = plyvel.DB(config.get("blockIndexDb"))
blockParser = BlockParser()
contract.initContractParser()
for blockNum in range(config.get("start_num"), config.get("end_num")):
try:
if count % 1000 == 0:
end = datetime.datetime.now()
delta = (end - start).total_seconds() * 1000000
logger.info("已处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count,
delta,
delta / count,
))
transWriter.flush()
blockHashBytes = blockIndexDb.get(num2Bytes(int(blockNum)))
blockBytes = blockDb.get(blockHashBytes)
blockHash = bytes2HexStr(blockHashBytes)
block = Tron_pb2.Block()
block.ParseFromString(blockBytes)
appendData = {
"block_num": blockNum,
"hash": blockHash,
}
ret = blockParser.Parse(transWriter, block, appendData)
if not ret:
logger.error(
"Failed to parse block num: {}".format(blockNum))
count += 1
except Exception:
logger.error("Failed to parse block num: {}".format(blockNum))
traceback.print_exc()
break
except Exception:
traceback.print_exc()
finally:
transWriter.flush()
transWriter.close()
end = datetime.datetime.now()
delta = (end - start).total_seconds() * 1000000
logger.info("共处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count - 1,
delta,
delta / (count - 1),
))
logger.info("处理 {} 个区块，预计用时 {} 小时".format(
total, ((total / (count - 1)) * delta) / 1000000 / 3600))
logger.info("开始时间: {}".format(start.strftime("%Y-%m-%d %H:%M:%S")))
logger.info("结束时间: {}".format(end.strftime("%Y-%m-%d %H:%M:%S")))
def main():
"""
- 1. 获取err csv目录
- 2. 读取并解析csv，| 分隔
- 3. 通过indexdb获取hash，再获取block，通过transid查找对应交易
- 4. 不再解析block和其它数据
5. trans中的部分autoDecode列解析为hex str
"""
import sys
if len(sys.argv) < 2:
logging.error("Please set input csv path")
return
inputDir = sys.argv[1]
if not os.path.isfile(inputDir):
logging.error("Dir not exists or is not file")
return
if not os.access(inputDir, os.R_OK):
logging.error("Can not read inputfile")
return
config, err = TransConfigParser.Parse()
if err is not None:
logger.error("Failed to get block config: {}".format(err))
exit(-1)
transWriter = TransWriter(config, tables)
start = datetime.datetime.now()
count = 1
with open(inputDir) as f:
f_csv = csv.reader(f, delimiter="|")
try:
blockDb = plyvel.DB(config.get("blockDb"))
blockIndexDb = plyvel.DB(config.get("blockIndexDb"))
contract.initContractParser()
for row in f_csv:
# print("row: ", row)
blockNum = row[1]
# print("blockNum type: ", type(blockNum))
transId = row[0]
try:
if count % 1000 == 0:
end = datetime.datetime.now()
logger.info(
"已处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count,
(end - start).microseconds,
(end - start).microseconds / count,
))
transWriter.flush()
blockHashBytes = blockIndexDb.get(num2Bytes(int(blockNum)))
blockBytes = blockDb.get(blockHashBytes)
blockHash = bytes2HexStr(blockHashBytes)
block = Tron_pb2.Block()
block.ParseFromString(blockBytes)
appendData = {
"block_num": blockNum,
"hash": blockHash,
"trans_id": transId,
}
bp = BlockParser()
ret = bp.Parse(transWriter, block, appendData)
if not ret:
logger.error(
"Failed to parse block num: {}".format(blockNum))
break
count += 1
except Exception:
logger.error(
"Failed to parse block num: {}".format(blockNum))
traceback.print_exc()
break
except Exception:
traceback.print_exc()
finally:
transWriter.flush()
transWriter.close()
end = datetime.datetime.now()
logger.info("共处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count - 1,
(end - start).microseconds,
(end - start).microseconds / (count - 1),
))
logger.info("处理 29617377 个区块，预计用时 {} 小时".format(
((29617377 / (count - 1)) *
(end - start).microseconds) / 1000000 / 3600))
logger.info("开始时间: {}".format(start.strftime("%Y-%m-%d %H:%M:%S")))
logger.info("结束时间: {}".format(end.strftime("%Y-%m-%d %H:%M:%S")))
def main():
# 读配置文件
# 连接数据库
# 开始解析
tables = [
"trans_ret",
"trans_ret_contract_result",
"trans_ret_log",
"trans_ret_log_topics",
"trans_ret_inter_trans",
"trans_ret_inter_trans_call_value",
"trans_ret_order_detail",
"trans_ret_error",
]
config, err = TransRetConfigParser.Parse()
if err is not None:
logger.error("Failed to get hawq config: {}".format(err))
exit(-1)
transWriter = TransWriter(config, tables=tables)
start = datetime.datetime.now()
count = 1
transInfoParser = TransactionInfoParser()
try:
transRetDb = plyvel.DB(config.get("input_dir"))
for i in range(config.get("start_num"), config.get("end_num")):
try:
if count % 1000 == 0:
end = datetime.datetime.now()
logger.info("已处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count,
(end - start).microseconds,
(end - start).microseconds / count,
))
transWriter.flush()
transRetBytes = transRetDb.get(num2Bytes(i))
if transRetBytes is None:
transWriter.write("trans_ret_error",
["block", i, "block not exists"])
continue
transRet = Tron_pb2.TransactionRet()
transRet.ParseFromString(transRetBytes)
for transInfo in transRet.transactioninfo:
transId = transInfoParser.colIndex[0].oc.getattr(transInfo)
ret = transInfoParser.Parse(transWriter, transInfo, None)
if not ret:
transWriter.write("trans_ret_error",
["trans", transId, None])
logger.error("Failed to parse block num: {}".format(i))
break
count += 1
except Exception:
logger.error("Failed to parse block num: {}".format(i))
traceback.print_exc()
transWriter.write("trans_ret_error", ["block", i, None])
break
# except Exception as e:
# logger.error("Failed to parse block num: {}".format(i))
# traceback.print_exc()
# raise e
# break
# end = datetime.datetime.now()
# logger.info(
# "共处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
# count - 1,
# (end - start).microseconds,
# (end - start).microseconds / (count - 1),
# )
# )
except Exception:
traceback.print_exc()
finally:
transWriter.flush()
transWriter.close()
end = datetime.datetime.now()
logger.info("共处理 {} 个区块，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count - 1,
(end - start).microseconds,
(end - start).microseconds / (count - 1),
))
logger.info("处理 29617377 个区块，预计用时 {} 小时".format(
((29617377 / (count - 1)) *
(end - start).microseconds) / 1000000 / 3600))
logger.info("开始时间: {}".format(start.strftime("%Y-%m-%d %H:%M:%S")))
logger.info("结束时间: {}".format(end.strftime("%Y-%m-%d %H:%M:%S")))