def divide(args):
'''
%prog divide input_dir output_dir_prefix
'''
p = OptionParser(divide.__doc__)
p.add_option('--pattern', default='*.jpg',
help='file name pattern')
p.add_option('--nimgs_per_folder', type='int', default=700,
help='~ number of images (<1000) in each smaller folder')
opts, args = p.parse_args(args)
if len(args) == 0:
sys.exit(not p.print_help())
input_dir, out_prefix, = args
df = GenDataFrameFromPath(Path(input_dir), pattern=opts.pattern)
n_folders = math.ceil(df.shape[0]/opts.nimgs_per_folder)
print('%s will be divided to %s datasets'%(df.shape[0], n_folders))
n = 0
for _, grp in cutlist(df['fnpath'].values, n_folders):
n += 1
output_folder = Path('%s_%s'%(out_prefix,n))
print(output_folder, grp.shape[0])
if not output_folder.exists():
output_folder.mkdir()
for i in grp:
copyfile(i, output_folder/i.name)
def PredictSlurmGPU(args):
"""
%prog model_name npyPattern("CM*.npy") job_n
generate prediction GPU jobs for all npy files
"""
p = OptionParser(PredictSlurmGPU.__doc__)
p.set_slurm_opts(jn=True)
opts, args = p.parse_args(args)
if len(args) == 0:
sys.exit(not p.print_help())
mn, npy_pattern, jobn, = args
if opts.prefix == 'myjob':
print('specify job name prefix!')
sys.exit()
npys = glob(npy_pattern)
print(len(npys))
grps = cutlist(npys, int(jobn))
for gn, grp in grps:
st, ed = gn.split('-')
ed = int(ed) + 1
gn = '%s-%s' % (st, ed)
cmd = "python -m schnablelab.CNN.Predict Predict %s '%s' %s\n" % (
mn, npy_pattern, gn)
opt = '%s.%s' % (opts.prefix, gn)
header = Slurm_gpu_header % (opts.time, opts.memory, opt, opt, opt)
header += "ml anaconda\nsource activate MCY\n"
header += cmd
with open('%s.gpu.slurm' % opt, 'w') as f:
f.write(header)
print('%s.gpu.slurm prediction GPU job file generated!' % opt)
def PlantHullBatch(args):
"""
%prog PlantHullBatch Pattern("*.png") job_n
generate PlantHull jobs for all image files
"""
p = OptionParser(PlantHullBatch.__doc__)
p.add_option('--mode',
default='real',
choices=['real', 'simu'],
help="real image or simulated image.")
p.set_slurm_opts()
opts, args = p.parse_args(args)
if len(args) == 0:
sys.exit(not p.print_help())
pattern, jobn, = args
all_imgs = glob(pattern)
all_cmds = []
for img in all_imgs:
imgpath = Path(img)
outpre = str(imgpath.stem)
cmd = 'python -m schnablelab.ImgPros.Preprocess PlantHull %s --crop True --segmentation True --border 80,10,10,10\n' % (img) \
if opts.mode=='real' \
else 'python -m schnablelab.ImgPros.Preprocess PlantHull %s --border 0,40,10,0 --thresh_cutoff 160\n' % (img)
print(cmd)
all_cmds.append(cmd)
grps = cutlist(all_cmds, int(jobn))
for gn, grp in grps:
header = Slurm_header % (opts.time, opts.memory, gn, gn, gn)
header += "ml anaconda\nsource activate MCY\n"
for cmd in grp:
header += cmd
jobname = '%s.ppnum.slurm' % (gn)
jobfile = open(jobname, 'w')
jobfile.write(header)
jobfile.close()
print('%s job file generated!' % jobname)