def run(args):
src_reader = NumpyReader(args.src_scp) if args.src == "npy" else MatReader(
args.src_scp, args.key)
num_mat = 0
mat_list = []
ops = args.op.split(",")
for op in ops:
if op and op not in supported_op:
raise RuntimeError(f"Unknown operation: {op}")
stack = "stack" in ops
with ArchiveWriter(args.dst_ark, args.scp) as writer:
for key, mat in src_reader:
for op in ops:
if op == "trans":
mat = np.transpose(mat)
elif op == "log":
mat = np.log(np.maximum(mat, EPSILON))
elif op == "minus":
mat = 1 - mat
else:
pass
if stack:
mat_list.append(mat)
else:
writer.write(key, mat)
num_mat += 1
if stack:
mat = np.vstack(mat_list)
writer.write(filekey(args.dst_ark), mat)
logger.info("Merge {0} matrix into archive {1}, shape as "
"{2[0]}x{2[1]}".format(num_mat, args.dst_ark,
mat.shape))
if not stack:
logger.info("Copy {0} matrix into archive {1}".format(
num_mat, args.dst_ark))
def run(args):
src_reader = NumpyReader(args.src_scp) if not args.matlab else MatReader(
args.src_scp, args.key)
num_mat = 0
mat_list = []
with ArchiveWriter(args.dst_ark, args.scp) as writer:
for key, mat in src_reader:
if args.transpose:
mat = np.transpose(mat)
if args.apply_log:
mat = np.log(np.maximum(mat, EPSILON))
if args.minus_by_one:
mat = 1 - mat
if not args.merge:
writer.write(key, mat)
else:
mat_list.append(mat)
num_mat += 1
if args.merge:
mat = np.vstack(mat_list)
writer.write(filekey(args.dst_ark), mat)
logger.info(
"Merge {0} matrix into archive {1}, shape as {2[0]}x{2[1]}".
format(num_mat, args.dst_ark, mat.shape))
if not args.merge:
logger.info("Copy {0} matrix into archive {1}".format(
num_mat, args.dst_ark))
def __init__(self, src_dir):
if not os.path.isdir(src_dir):
raise RuntimeError("NumpyReader expect dir as input")
flist = glob.glob(os.path.join(src_dir, "*.npy"))
self.index_dict = {filekey(f): f for f in flist}
