def main(): # if len(os.Args) < 2: # logging.error( # "请输入account数据库位置", # ) # accountDB = plyvel.DB(os.Args[1]) config, err = ConfigParser.Parse() if err is not None: logging.error("Failed to get hawq config: {}".format(err)) exit(-1) conn = getConn(config) if not conn: logging.error("Failed to connect hawq : {}".format(err)) exit(-1) try: cur = conn.cursor() accountDB = plyvel.DB("/data2/20210425/output-directory/database/account") # accountIt = accountDB.iterator() accInsert = CommonParseAndInsert( cursor=cur, cols=Account.cols, subCols=Account.subCols, table=Account.table, ) count = 0 start = datetime.datetime.now() for k, v in accountDB: if count % 10000 == 0: end = datetime.datetime.now() logging.info( "已处理 {} 个账户,共耗时 {} 微秒".format(count, (end - start).microseconds) ) acc = Tron_pb2.Account() acc.ParseFromString(v) ret, sqls = accInsert.Insert(acc) # logging.info("sqls: {}".format(sqls)) if not ret or len(sqls) == 0: logging.error("===================") logging.error("解析插入失败:\n address hex: {}".format(b2hs(acc.address))) logging.error( "解析插入失败:\n address: {}".format(addressFromBytes(acc.address)) ) logging.error("===================\n\n\n") continue cur.execute("".join(sqls)) conn.commit() count += 1 except Exception as e: traceback.print_exc() logging.error("Failed to run main: {}".format(e)) finally: try: conn.rollback() finally: conn.close()
def test(): config, err = ConfigParser.Parse() if err is not None: logging.error("Failed to get hawq config: {}".format(err)) exit(-1) conn = getConn(config) if not conn: logging.error("Failed to connect hawq : {}".format(err)) exit(-1) try: cur = conn.cursor() accountDB = plyvel.DB("/data2/20210425/output-directory/database/account") # accountIt = accountDB.iterator() accInsert = CommonParseAndInsert( cursor=cur, cols=Account.cols, subCols=Account.subCols, table=Account.table, ) v = accountDB.get(bytes.fromhex("4100001f9ac7032955f71612dea92dc850ff3fa087")) acc = Tron_pb2.Account() acc.ParseFromString(v) ret, sqls = accInsert.Insert(acc) # logging.info("sqls: {}".format(sqls)) if not ret or len(sqls) == 0: logging.error("===================") logging.error("解析插入失败:\n address hex: {}".format(b2hs(acc.address))) logging.error("解析插入失败:\n address: {}".format(addressFromBytes(acc.address))) logging.error("===================\n\n\n") except Exception as e: traceback.print_exc() logging.error("Failed to run main: {}".format(e)) finally: try: conn.rollback() finally: conn.close()
def main(): config, err = ConfigParser.Parse() if err is not None: logging.error("Failed to get hawq config: {}".format(err)) exit(-1) tableWriter = TableWriter(config) start = datetime.datetime.now() try: accountDB = plyvel.DB(config.get("input_dir")) # accountIt = accountDB.iterator() accInsert = CommonParseAndInsert( cursor=tableWriter, cols=Account.cols, subCols=Account.subCols, table=Account.table, ) count = 0 acc = Tron_pb2.Account() for k, v in accountDB: count += 1 try: # if count < 180000: # continue if count % 10000 == 0: tableWriter.flush() end = datetime.datetime.now() logging.info("已处理 {} 个账户,共耗时 {} 微秒".format( count, (end - start).microseconds)) acc.ParseFromString(v) # if addressFromBytes(acc.address) not in [ # # "TA7CEh4xHiY8kh28D6nyM2zVYKB8PSbZhh", # # "T9yDMSrP8exVeYbBy7yFYnM5BNYbquSGRp", # "T9zoNbweZZXd7eVxbasQXDth54FEU3xsnb", # "T9ztt6FVT6c6mSGGRvbBDC32xqtxRAnQQ9", # "TA4JFjPAaY8Gtr4qUVUqBxK98aewUVEPAo", # "TA55VXkWcF2EbKX64aXiP21NidLrvAtF4d", # "TA5k7U2uUz6MxF3bnX1DE6KyzAX63MQ9tp", # ]: # continue ret = accInsert.Insert(tableWriter, acc) if not ret: tableWriter.write( "error_account", [ count, b2hs(acc.address), addressFromBytes(acc.address) ], ) logging.error("解析插入失败:\n address hex: {}".format( b2hs(acc.address))) logging.error("解析插入失败:\n address: {}".format( addressFromBytes(acc.address))) except Exception as e: logging.error("解析插入失败:\n address hex: {}".format( b2hs(acc.address))) logging.error("解析插入失败:\n address: {}".format( addressFromBytes(acc.address))) tableWriter.write( "error_account", [count, b2hs(acc.address), addressFromBytes(acc.address)], ) logger.error("解析插入失败原因: {}".format(e)) except Exception as e: traceback.print_exc() logging.error("Failed to run main: {}".format(e)) finally: tableWriter.close() end = datetime.datetime.now() logging.info("共处理 {} 个账户,共耗时 {} 微秒, 平均单个耗时 {} 微秒".format( count - 1, (end - start).microseconds, (end - start).microseconds / (count - 1), )) logging.info("处理 30004228 个账户,预计用时 {} 小时".format( (((30004228 / (count - 1)) * (end - start).microseconds)) / 1000000 / 3600)) logging.info("开始时间: {}".format(start.strftime("%Y-%m-%d %H:%M:%S"))) logging.info("结束时间: {}".format(end.strftime("%Y-%m-%d %H:%M:%S")))
def addressFromBytes(addr): cache = tronapi.common.account.Address().from_hex(bytes.decode(b2hs(addr))) if type(cache) == bytes: return cache.decode() return cache
def bytes2HexStr(data): return b2hs(data).decode()
def addressFromBytes(addr): return tronapi.common.account.Address().from_hex(bytes.decode(b2hs(addr))).decode()