def combine_infer(source, target, prefix, pwx, pwy):
if isinstance(source, (list, tuple)):
source = source[0]
source = os.path.abspath(source)
print("COMBINE INFER IN:", source)
files = os.listdir(source)
filesfull = []
for file_ in files:
fullname = os.path.join(source, file_)
if os.path.isdir(fullname):
continue
prefix_tmp, _, _ = utg.seperate_file_name(file_)
if prefix_tmp != prefix:
continue
filesfull.append(fullname)
tmp = np.array(np.load(filesfull[0]))
_, height, width, _ = tmp.shape
frames = len(filesfull)
height -= pwx * 2
width -= pwy * 2
sino3d = np.zeros([width, height, frames])
for i in xrange(frames):
filename = prefix + "%09d.npy" % i
fullname = os.path.join(source, filename)
sino2d = np.load(fullname)
for ix in xrange(height):
for iy in xrange(width):
sino3d[iy, ix, i] = sino2d[0, ix + pwx, iy + pwy, 0]
savename = os.path.join(os.path.abspath(target), 'srresult.npy')
np.save(savename, sino3d)
def print_std_filename(file):
if isinstance(file, (list, tuple)):
map(print_std_filename, file)
prefix, id_, suffix = utg.seperate_file_name(file)
print("Analysic filename:{}".format(file))
print("+P:{0}, I:{1}, S:{2}.".format(prefix, id_, suffix))
name_std = utg.form_file_name(prefix, id_, suffix)
print("+STD:{}".format(name_std))
def merge(source, target, random_rename=True, copy_file=False, no_action=False, recurrent=False, prefix=None, suffix=None):
print("merge tool called on {0}".format(source))
if random_rename:
rename(source, prefix=None, suffix=suffix,
recurrent=recurrent, no_action=no_action)
if isinstance(source, (list, tuple)):
map(lambda folder: merge(folder, target, random_rename, copy_file, no_action, recurrent, prefix, suffix),
source)
return None
folder_name = os.path.abspath(source)
target_full = os.path.abspath(target)
files_maybe = os.listdir(folder_name)
files_full = list(map(lambda filename: os.path.join(
folder_name, filename), files_maybe))
files = list(filter(os.path.isfile, files_full))
files = list(map(os.path.basename, files))
if suffix is not None:
for file_old in files:
_, _, suffix_old = utg.seperate_file_name(file_old)
if suffix_old != suffix:
files.remove(file_old)
dirs = list(filter(os.path.isdir, files_full))
dirs = list(map(os.path.basename, dirs))
if recurrent:
for dir_ in dirs:
fullpath = os.path.join(folder_name, dir_)
merge(fullpath, target_full, random_rename, copy_file,
no_action, recurrent, prefix, suffix)
for file_old in files:
full_old = os.path.join(folder_name, file_old)
full_new = os.path.join(target_full, file_old)
action = "COPY" if copy_file else "MOVE"
if no_action:
print("{0}: {1} TO: {2}.".format(action, full_old, full_new))
else:
if copy_file:
shutil.copyfile(full_old, full_new)
else:
os.rename(full_old, full_new)
def img2jpg(folder_name, prefix, ids=None):
"""Convert all image files under a folder into jpg files."""
path = os.path.abspath(folder_name)
files = os.listdir(path)
for file in files:
prefix, id, suffix = utg.seperate_file_name(file)
if suffix == 'jpg':
continue
if ids is not None and id not in ids:
continue
fullname = os.path.join(path, file)
img = uti.imread(fullname)
if img.mode != 'RGB':
img.convert('RGB')
newname = utg.form_file_name(prefix, id, 'jpg')
fullnamenew = os.path.join(path, newname)
img.save(fullnamenew, 'JPEG')
def __init__(self,
folder,
prefix,
ids=None,
random_shuffle=False,
suffix='npy',
name='FileNameLooper',
max_epoch=1,
is_seal=False):
super(FileNameLooper, self).__init__(is_start=True,
name=name,
is_seal=is_seal)
self._output_type = str
self._path = os.path.abspath(folder)
self._is_random = random_shuffle
self._prefix = prefix
self._suffix = suffix
list_all = os.listdir(self._path)
list_all.sort()
# file name filter
self._file_names = []
if ids is None:
for file in list_all:
prefix, id, suffix = utg.seperate_file_name(file)
if prefix is not None:
self._file_names.append(file)
else:
for id_ in ids:
filename = utg.form_file_name(self._prefix, id_, self._suffix)
if filename in list_all:
self._file_names.append(filename)
# counters
self._epoch_counter = Counter(max_epoch)
self._fid_counter = Counter(len(self._file_names))
self._is_random = random_shuffle
self._new_epoch()
self._cache = None
def rename(source, prefix=None, suffix=None, recurrent=False, no_action=False, force=False):
"""Rename all files in a folder, reform prefixes, with filter using suffix.
Only rename files with form: name[.suffix].
If prefix is given, then performing prefix changing task. All files are
suppoosed to be in standard name form.
If prefix is None, a random prefix will be added.
"""
folder_name = source
if isinstance(folder_name, (list, tuple)):
for folder in folder_name:
rename(folder, prefix, suffix, recurrent, no_action, force)
return None
folder_name = os.path.abspath(folder_name)
files_maybe = os.listdir(folder_name)
files_full = list(map(lambda filename: os.path.join(
folder_name, filename), files_maybe))
files = list(filter(os.path.isfile, files_full))
files = list(map(os.path.basename, files))
if suffix is not None and not force:
for file_old in files:
_, _, suffix_old = utg.seperate_file_name(file_old)
if suffix_old != suffix:
files.remove(file_old)
dirs = filter(os.path.isdir, files_full)
dirs = list(map(os.path.basename, dirs))
if recurrent:
for dir_ in dirs:
fullpath = os.path.join(folder_name, dir_)
rename(fullpath, prefix, suffix, recurrent, no_action)
if prefix is None:
print("RENAME>>>RANDOM PREFIX: {}".format(folder_name))
pipe_random_prefix = utp.RandomPrefix()
for file_old in files:
prefix = next(pipe_random_prefix.out)
if suffix is '':
file_new = prefix
else:
file_new = prefix + '.' + suffix
full_old = os.path.join(folder_name, file_old)
full_new = os.path.join(folder_name, file_new)
if no_action:
print("rename: {0} TO: {1}.".format(full_old, full_new))
else:
os.rename(full_old, full_new)
else:
print("RENAME>>>FORMAL RENAME: {}".format(folder_name))
cid = 0
for file_old in files:
_, _, suffix_old = utg.seperate_file_name(file_old)
file_new = utg.form_file_name(prefix, cid, suffix_old)
full_old = os.path.join(folder_name, file_old)
full_new = os.path.join(folder_name, file_new)
if no_action:
print("rename: {0} TO: {1}.".format(full_old, full_new))
else:
os.rename(full_old, full_new)
cid += 1
def main(argv):
"""command line support.
"""
print('Dataset tools for command line.')
parser = argparse.ArgumentParser(description='Dataset construction.')
parser.add_argument('--rename',
action='store_true',
default=False,
help='rename all data items.')
parser.add_argument('--merge',
action='store_true',
default=False,
help='merge multiple folders.')
parser.add_argument('--shuffle',
action='store_true',
default=False,
help='shuffle data file ids.')
parser.add_argument('--img2jpg',
action='store_true',
default=False,
help='convert all images to JPEG.')
parser.add_argument('--sino3d2sino2d',
action='store_true',
default=False,
help='Convert 3D sinograms to multiple 2D sinograms.')
parser.add_argument('--combine_infer',
action='store_true',
default=False,
help='combine inference sino2d to sino3d.')
parser.add_argument('--jpg2npy',
action='store_true',
default=False,
help='convert all JPEG to npy.')
parser.add_argument('--raw2npy',
action='store_true',
default=False,
help='read raw data to npy')
parser.add_argument('--filename',
action='store_true',
default=False,
help='Analysis file name.')
parser.add_argument('--print',
action='store_true',
default=False,
help='print args, do nothing.')
parser.add_argument('--pad_sino',
action='store_true',
default=False,
help='padding sinograms.')
parser.add_argument('--no_action', '-n',
action='store_true',
default=False,
help='print settings, do noting.')
parser.add_argument('--force', '-f',
action='store_true',
default=False,
help='force doing.')
parser.add_argument('--recurrent', '-r',
action='store_true',
default=False,
help='recurrent on sub folders.')
parser.add_argument('--copy_file',
action='store_true',
default=False,
help='copy files instead of move files.')
parser.add_argument('--random_rename',
action='store_true',
default=False,
help='Add random prefix before merge.')
parser.add_argument('--source', '-s', dest='source', required=True, nargs='+',
help='source folder/file.')
parser.add_argument('--target', '-t',
dest='target', default=None,
help='target folder/file.')
parser.add_argument('--suffix', dest='suffix', default=None, help='suffix')
parser.add_argument('--prefix', dest='prefix', default=None, help='prefix')
parser.add_argument('--prefix_new', dest='prefix_new',
default=None, help='new prefix')
parser.add_argument('--shape', dest='shape', nargs='+', type=int)
parser.add_argument('--index0', dest='id0', type=int)
parser.add_argument('--index1', dest='id1', type=int)
parser.add_argument('--endian', dest='endian',
default='l', help='Endianness, l or b')
parser.add_argument('--pwx', type=int, help='padding window x')
parser.add_argument('--pwy', type=int, help='padding window y')
args = parser.parse_args(argv)
if args.print:
print(args)
return
if args.filename:
for filename in args.source:
print(utg.seperate_file_name(filename))
return
if args.rename:
rename(args.source, args.prefix, args.suffix,
args.recurrent, args.no_action, args.force)
if args.merge:
merge(args.source, args.target, args.random_rename, args.copy_file,
args.no_action, args.recurrent, args.prefix, args.suffix)
if args.shuffle:
shuffle(args.source, args.target, args.prefix,
args.suffix, args.no_action)
if args.pad_sino:
pad_sino(args.source, args.target, args.prefix, args.pwx, args.pwy)
if args.combine_infer:
combine_infer(source=args.source, target=args.target,
prefix=args.prefix, pwx=args.pwx, pwy=args.pwy)
# if args.img2jpg:
# img2jpeg(args.source, args.prefix)
# if args.jpg2npy:
# jpg2npy(args.source, args.prefix, args.id0, args.id1)
if args.raw2npy:
raw2npy(args.source, shape=args.shape, prefix=args.prefix,
suffix=args.suffix, id0=args.id0, id1=args.id1)
if args.sino3d2sino2d:
sino3d2sino2d(args.source, args.target, args.prefix)
def raw2npy(folder_name, shape, prefix, **kwargs):
"""convert all raw files into npy files.
"""
if isinstance(folder_name, (list, tuple)):
for folder in folder_name:
raw2npy(folder, shape, prefix, **kwargs)
return None
path = os.path.abspath(folder_name)
files = os.listdir(path)
if 'suffix' in kwargs:
suffix = kwargs['suffix']
else:
suffix = ''
if 'id0' in kwargs:
id0 = kwargs['id0']
else:
id0 = None
if id0 is None:
id0 = 0
if 'id1' in kwargs:
id1 = kwargs['id1']
else:
id1 = None
if id1 is None:
id1 = len(files)
ids = list(xrange(int(id0), int(id1) + 1))
for file in files:
fullname = os.path.join(path, file)
if os.path.isdir(fullname):
continue
prefix_f, id_f, suffix_f = utg.seperate_file_name(file)
if prefix_f != prefix:
continue
if suffix_f != suffix:
continue
if id_f not in ids:
continue
print("processing: {}.".format(fullname))
if shape is None:
raise ValueError('shape is not defined.')
height = shape[0]
width = shape[1]
frame = shape[2]
pixel = height * width * frame
data = np.zeros([pixel])
with open(fullname) as f:
bindata = f.read()
for i in xrange(pixel):
res = struct.unpack('<f', bindata[i * 4:i * 4 + 4])
data[i] = res[0]
data2 = np.zeros([height, width, frame])
for i in range(width):
for j in range(height):
for k in range(frame):
idf = i + j * width + k * width * height
data2[j, i, k] = data[idf]
filename = utg.form_file_name(prefix_f, id_f, 'npy')
fullname = os.path.join(folder_name, filename)
np.save(fullname, data2)
def test_basic_0(self):
output = utg.seperate_file_name("sino000000018.raw")
expect = ('sino', 18, 'raw')
self.assertTrue(output == expect, msg=utg.errmsg(output, expect))