def main():
# if len(os.Args) < 2:
# logging.error(
# "请输入account数据库位置",
# )
# accountDB = plyvel.DB(os.Args[1])
config, err = ConfigParser.Parse()
if err is not None:
logging.error("Failed to get hawq config: {}".format(err))
exit(-1)
conn = getConn(config)
if not conn:
logging.error("Failed to connect hawq : {}".format(err))
exit(-1)
try:
cur = conn.cursor()
accountDB = plyvel.DB("/data2/20210425/output-directory/database/account")
# accountIt = accountDB.iterator()
accInsert = CommonParseAndInsert(
cursor=cur,
cols=Account.cols,
subCols=Account.subCols,
table=Account.table,
)
count = 0
start = datetime.datetime.now()
for k, v in accountDB:
if count % 10000 == 0:
end = datetime.datetime.now()
logging.info(
"已处理 {} 个账户，共耗时 {} 微秒".format(count, (end - start).microseconds)
)
acc = Tron_pb2.Account()
acc.ParseFromString(v)
ret, sqls = accInsert.Insert(acc)
# logging.info("sqls: {}".format(sqls))
if not ret or len(sqls) == 0:
logging.error("===================")
logging.error("解析插入失败:\n address hex: {}".format(b2hs(acc.address)))
logging.error(
"解析插入失败:\n address: {}".format(addressFromBytes(acc.address))
)
logging.error("===================\n\n\n")
continue
cur.execute("".join(sqls))
conn.commit()
count += 1
except Exception as e:
traceback.print_exc()
logging.error("Failed to run main: {}".format(e))
finally:
try:
conn.rollback()
finally:
conn.close()
def test():
config, err = ConfigParser.Parse()
if err is not None:
logging.error("Failed to get hawq config: {}".format(err))
exit(-1)
conn = getConn(config)
if not conn:
logging.error("Failed to connect hawq : {}".format(err))
exit(-1)
try:
cur = conn.cursor()
accountDB = plyvel.DB("/data2/20210425/output-directory/database/account")
# accountIt = accountDB.iterator()
accInsert = CommonParseAndInsert(
cursor=cur,
cols=Account.cols,
subCols=Account.subCols,
table=Account.table,
)
v = accountDB.get(bytes.fromhex("4100001f9ac7032955f71612dea92dc850ff3fa087"))
acc = Tron_pb2.Account()
acc.ParseFromString(v)
ret, sqls = accInsert.Insert(acc)
# logging.info("sqls: {}".format(sqls))
if not ret or len(sqls) == 0:
logging.error("===================")
logging.error("解析插入失败:\n address hex: {}".format(b2hs(acc.address)))
logging.error("解析插入失败:\n address: {}".format(addressFromBytes(acc.address)))
logging.error("===================\n\n\n")
except Exception as e:
traceback.print_exc()
logging.error("Failed to run main: {}".format(e))
finally:
try:
conn.rollback()
finally:
conn.close()
def main():
config, err = ConfigParser.Parse()
if err is not None:
logging.error("Failed to get hawq config: {}".format(err))
exit(-1)
tableWriter = TableWriter(config)
start = datetime.datetime.now()
try:
accountDB = plyvel.DB(config.get("input_dir"))
# accountIt = accountDB.iterator()
accInsert = CommonParseAndInsert(
cursor=tableWriter,
cols=Account.cols,
subCols=Account.subCols,
table=Account.table,
)
count = 0
acc = Tron_pb2.Account()
for k, v in accountDB:
count += 1
try:
# if count < 180000:
# continue
if count % 10000 == 0:
tableWriter.flush()
end = datetime.datetime.now()
logging.info("已处理 {} 个账户，共耗时 {} 微秒".format(
count, (end - start).microseconds))
acc.ParseFromString(v)
# if addressFromBytes(acc.address) not in [
# # "TA7CEh4xHiY8kh28D6nyM2zVYKB8PSbZhh",
# # "T9yDMSrP8exVeYbBy7yFYnM5BNYbquSGRp",
# "T9zoNbweZZXd7eVxbasQXDth54FEU3xsnb",
# "T9ztt6FVT6c6mSGGRvbBDC32xqtxRAnQQ9",
# "TA4JFjPAaY8Gtr4qUVUqBxK98aewUVEPAo",
# "TA55VXkWcF2EbKX64aXiP21NidLrvAtF4d",
# "TA5k7U2uUz6MxF3bnX1DE6KyzAX63MQ9tp",
# ]:
# continue
ret = accInsert.Insert(tableWriter, acc)
if not ret:
tableWriter.write(
"error_account",
[
count,
b2hs(acc.address),
addressFromBytes(acc.address)
],
)
logging.error("解析插入失败:\n address hex: {}".format(
b2hs(acc.address)))
logging.error("解析插入失败:\n address: {}".format(
addressFromBytes(acc.address)))
except Exception as e:
logging.error("解析插入失败:\n address hex: {}".format(
b2hs(acc.address)))
logging.error("解析插入失败:\n address: {}".format(
addressFromBytes(acc.address)))
tableWriter.write(
"error_account",
[count,
b2hs(acc.address),
addressFromBytes(acc.address)],
)
logger.error("解析插入失败原因: {}".format(e))
except Exception as e:
traceback.print_exc()
logging.error("Failed to run main: {}".format(e))
finally:
tableWriter.close()
end = datetime.datetime.now()
logging.info("共处理 {} 个账户，共耗时 {} 微秒, 平均单个耗时 {} 微秒".format(
count - 1,
(end - start).microseconds,
(end - start).microseconds / (count - 1),
))
logging.info("处理 30004228 个账户，预计用时 {} 小时".format(
(((30004228 / (count - 1)) *
(end - start).microseconds)) / 1000000 / 3600))
logging.info("开始时间: {}".format(start.strftime("%Y-%m-%d %H:%M:%S")))
logging.info("结束时间: {}".format(end.strftime("%Y-%m-%d %H:%M:%S")))
def addressFromBytes(addr):
cache = tronapi.common.account.Address().from_hex(bytes.decode(b2hs(addr)))
if type(cache) == bytes:
return cache.decode()
return cache
def bytes2HexStr(data):
return b2hs(data).decode()
def addressFromBytes(addr):
return tronapi.common.account.Address().from_hex(bytes.decode(b2hs(addr))).decode()