def v1like_extract_fromcsv( config_fname, input_csv_fname, output_suffix, input_path=DEFAULT_INPUT_PATH, nprocessors=DEFAULT_NPROCESSORS, overwrite=DEFAULT_OVERWRITE, ): assert (nprocessors >= 1) csvr = csv.reader(open(input_csv_fname)) rows = [row for row in csvr] fnames = sp.array([row[:-2] for row in rows]).ravel() #fnames.sort() sp.random.shuffle( fnames) # shuffle to enable multiple instances to run in // nfnames = len(fnames) # -- set up progress bar widgets = [ RotatingMarker(), " Progress: ", Percentage(), " ", Bar(left='[', right=']'), ' ', #" (", FilenameUpdate(fnames), ") ", ETA()] ETA() ] pbar = ProgressBar(widgets=widgets, maxval=nfnames) # - should we use multiprocessing ? if nprocessors > 1: # -- prepare all parameters import multiprocessing as mp pool = mp.Pool(nprocessors) print "Creating list of parameters for multiprocessing..." params = [] for fname in fnames: input_fname = path.join(input_path, fname) output_fname = input_fname + output_suffix params += [(config_fname, input_fname, output_fname, overwrite)] # -- async iterator map done = pool.imap_unordered(pv1like_extract, params) # -- update progress bar print "Processing %d images..." % (nfnames) i = 1 pbar.start() for _ in done: pbar.update(i) i += 1 else: # -- process images pbar.start() for i, fname in enumerate(fnames): input_fname = path.join(input_path, fname) output_fname = input_fname + output_suffix v1like_extract(config_fname, input_fname, output_fname, overwrite=overwrite) pbar.update(i + 1) pbar.finish() print
def v1like_extract_fromcsv(config_fname, input_csv_fname, output_suffix, input_path = DEFAULT_INPUT_PATH, nprocessors = DEFAULT_NPROCESSORS, overwrite = DEFAULT_OVERWRITE, ): assert(nprocessors >= 1) csvr = csv.reader(open(input_csv_fname)) rows = [ row for row in csvr ] fnames = sp.array([ row[:-2] for row in rows ]).ravel() #fnames.sort() sp.random.shuffle(fnames) # shuffle to enable multiple instances to run in // nfnames = len(fnames) # -- set up progress bar widgets = [RotatingMarker(), " Progress: ", Percentage(), " ", Bar(left='[',right=']'), ' ', #" (", FilenameUpdate(fnames), ") ", ETA()] ETA()] pbar = ProgressBar(widgets=widgets, maxval=nfnames) # - should we use multiprocessing ? if nprocessors > 1: # -- prepare all parameters import multiprocessing as mp pool = mp.Pool(nprocessors) print "Creating list of parameters for multiprocessing..." params = [] for fname in fnames: input_fname = path.join(input_path, fname) output_fname = input_fname + output_suffix params += [(config_fname, input_fname, output_fname, overwrite)] # -- async iterator map done = pool.imap_unordered(pv1like_extract, params) # -- update progress bar print "Processing %d images..." % (nfnames) i = 1 pbar.start() for _ in done: pbar.update(i) i += 1 else: # -- process images pbar.start() for i, fname in enumerate(fnames): input_fname = path.join(input_path, fname) output_fname = input_fname + output_suffix v1like_extract(config_fname, input_fname, output_fname, overwrite = overwrite) pbar.update(i+1) pbar.finish() print
def pv1like_extract(params): config_fname, input_fname, output_fname, overwrite = params return v1like_extract(config_fname, input_fname, output_fname, overwrite=overwrite)
def pv1like_extract(params): config_fname, input_fname, output_fname, overwrite = params return v1like_extract(config_fname, input_fname, output_fname, overwrite = overwrite)