def v1like_extract_fromcsv(
config_fname,
input_csv_fname,
output_suffix,
input_path=DEFAULT_INPUT_PATH,
nprocessors=DEFAULT_NPROCESSORS,
overwrite=DEFAULT_OVERWRITE,
):
assert (nprocessors >= 1)
csvr = csv.reader(open(input_csv_fname))
rows = [row for row in csvr]
fnames = sp.array([row[:-2] for row in rows]).ravel()
#fnames.sort()
sp.random.shuffle(
fnames) # shuffle to enable multiple instances to run in //
nfnames = len(fnames)
# -- set up progress bar
widgets = [
RotatingMarker(),
" Progress: ",
Percentage(),
" ",
Bar(left='[', right=']'),
' ',
#" (", FilenameUpdate(fnames), ") ", ETA()]
ETA()
]
pbar = ProgressBar(widgets=widgets, maxval=nfnames)
# - should we use multiprocessing ?
if nprocessors > 1:
# -- prepare all parameters
import multiprocessing as mp
pool = mp.Pool(nprocessors)
print "Creating list of parameters for multiprocessing..."
params = []
for fname in fnames:
input_fname = path.join(input_path, fname)
output_fname = input_fname + output_suffix
params += [(config_fname, input_fname, output_fname, overwrite)]
# -- async iterator map
done = pool.imap_unordered(pv1like_extract, params)
# -- update progress bar
print "Processing %d images..." % (nfnames)
i = 1
pbar.start()
for _ in done:
pbar.update(i)
i += 1
else:
# -- process images
pbar.start()
for i, fname in enumerate(fnames):
input_fname = path.join(input_path, fname)
output_fname = input_fname + output_suffix
v1like_extract(config_fname,
input_fname,
output_fname,
overwrite=overwrite)
pbar.update(i + 1)
pbar.finish()
print
def v1like_extract_fromcsv(config_fname,
input_csv_fname,
output_suffix,
input_path = DEFAULT_INPUT_PATH,
nprocessors = DEFAULT_NPROCESSORS,
overwrite = DEFAULT_OVERWRITE,
):
assert(nprocessors >= 1)
csvr = csv.reader(open(input_csv_fname))
rows = [ row for row in csvr ]
fnames = sp.array([ row[:-2] for row in rows ]).ravel()
#fnames.sort()
sp.random.shuffle(fnames) # shuffle to enable multiple instances to run in //
nfnames = len(fnames)
# -- set up progress bar
widgets = [RotatingMarker(), " Progress: ", Percentage(), " ",
Bar(left='[',right=']'), ' ',
#" (", FilenameUpdate(fnames), ") ", ETA()]
ETA()]
pbar = ProgressBar(widgets=widgets, maxval=nfnames)
# - should we use multiprocessing ?
if nprocessors > 1:
# -- prepare all parameters
import multiprocessing as mp
pool = mp.Pool(nprocessors)
print "Creating list of parameters for multiprocessing..."
params = []
for fname in fnames:
input_fname = path.join(input_path, fname)
output_fname = input_fname + output_suffix
params += [(config_fname,
input_fname,
output_fname,
overwrite)]
# -- async iterator map
done = pool.imap_unordered(pv1like_extract, params)
# -- update progress bar
print "Processing %d images..." % (nfnames)
i = 1
pbar.start()
for _ in done:
pbar.update(i)
i += 1
else:
# -- process images
pbar.start()
for i, fname in enumerate(fnames):
input_fname = path.join(input_path, fname)
output_fname = input_fname + output_suffix
v1like_extract(config_fname,
input_fname,
output_fname,
overwrite = overwrite)
pbar.update(i+1)
pbar.finish()
print
def pv1like_extract(params):
config_fname, input_fname, output_fname, overwrite = params
return v1like_extract(config_fname,
input_fname,
output_fname,
overwrite=overwrite)
def pv1like_extract(params):
config_fname, input_fname, output_fname, overwrite = params
return v1like_extract(config_fname,
input_fname,
output_fname,
overwrite = overwrite)
