def run(args): src_reader = NumpyReader(args.src_scp) if args.src == "npy" else MatReader( args.src_scp, args.key) num_mat = 0 mat_list = [] ops = args.op.split(",") for op in ops: if op and op not in supported_op: raise RuntimeError(f"Unknown operation: {op}") stack = "stack" in ops with ArchiveWriter(args.dst_ark, args.scp) as writer: for key, mat in src_reader: for op in ops: if op == "trans": mat = np.transpose(mat) elif op == "log": mat = np.log(np.maximum(mat, EPSILON)) elif op == "minus": mat = 1 - mat else: pass if stack: mat_list.append(mat) else: writer.write(key, mat) num_mat += 1 if stack: mat = np.vstack(mat_list) writer.write(filekey(args.dst_ark), mat) logger.info("Merge {0} matrix into archive {1}, shape as " "{2[0]}x{2[1]}".format(num_mat, args.dst_ark, mat.shape)) if not stack: logger.info("Copy {0} matrix into archive {1}".format( num_mat, args.dst_ark))
def run(args): src_reader = NumpyReader(args.src_scp) if not args.matlab else MatReader( args.src_scp, args.key) num_mat = 0 mat_list = [] with ArchiveWriter(args.dst_ark, args.scp) as writer: for key, mat in src_reader: if args.transpose: mat = np.transpose(mat) if args.apply_log: mat = np.log(np.maximum(mat, EPSILON)) if args.minus_by_one: mat = 1 - mat if not args.merge: writer.write(key, mat) else: mat_list.append(mat) num_mat += 1 if args.merge: mat = np.vstack(mat_list) writer.write(filekey(args.dst_ark), mat) logger.info( "Merge {0} matrix into archive {1}, shape as {2[0]}x{2[1]}". format(num_mat, args.dst_ark, mat.shape)) if not args.merge: logger.info("Copy {0} matrix into archive {1}".format( num_mat, args.dst_ark))
def __init__(self, src_dir): if not os.path.isdir(src_dir): raise RuntimeError("NumpyReader expect dir as input") flist = glob.glob(os.path.join(src_dir, "*.npy")) self.index_dict = {filekey(f): f for f in flist}