def run(self): global GLOBAL_ID h, w = self.img1.shape bb = [0, h-1, 0, w-1] # list MATCHES: [(imgath, score1, score2, patch, i1, i2, j1, j2, rszFac), ...] t = time.time() imgpatch = self.img1.astype('float32') / 255.0 if NPROCS == 1: print "(ThreadDoTempMatch): Using 1 process (TM_THRESHOLD={0}).".format(self.THRESHOLD) matches = shared.find_patch_matchesV1(imgpatch, bb[:], self.imgpaths, threshold=self.THRESHOLD, output_Ireg=False, save_patches=True, outdir=outdir, outid_start=GLOBAL_ID, jobid=jobid, progress_queue=progress_queue) else: print "(ThreadDoTempMatch): Using {0} processes. (TM_THRESHOLD={1})".format(NPROCS, self.THRESHOLD) procs = [] # [(Process p, int numimgs), ...] if progress_queue != None: manager = multiprocessing.Manager() else: manager = progress_queue._manager queue = manager.Queue() t_start = time.time() step = int(math.ceil(len(self.imgpaths) / float(NPROCS))) for procnum in xrange(NPROCS): i_start = procnum * step i_end = i_start + step if procnum == NPROCS - 1: imgpaths_in = self.imgpaths[i_start:] else: imgpaths_in = self.imgpaths[i_start:i_end] outdir_in = os.path.join(outdir, "proc_{0}".format(procnum)) p = multiprocessing.Process(target=mp_patchmatch, args=(queue, imgpatch, bb[:], imgpaths_in), kwargs={'output_Ireg': False, 'save_patches': True, 'outdir': outdir_in, 'outid_start': GLOBAL_ID, 'jobid': jobid, 'progress_queue': progress_queue}) p.start() procs.append((p, len(imgpaths_in))) matches = [] for i, (proc, numimgs) in enumerate(procs): proc.join() dur = time.time() - t_start p_matches = queue.get() print "(Process_{0}) Finished temp matching, got {1} matches ({2:.4f}s, {3} images total)".format(i, len(p_matches), dur, numimgs) matches.extend(p_matches) dur = time.time() - t print "DONE with temp matching. Found: {0} matches ({1:.4f}s, {2:.5f}s per image)".format(len(matches), dur, dur / float(len(self.imgpaths))) GLOBAL_ID += len(matches) matches_size = asizeof(matches) print " sizeof(matches): {0} bytes ({1:.4f} MB)".format(matches_size, matches_size / 1e6) wx.CallAfter(self.callback, matches)
def sizeit(thing): nbytes = asize.asizeof(thing) return "{0} bytes ({1} MB)".format(nbytes, nbytes / 1e6)
def decode_ballots(self, ballots, manager=None, queue=None, skipVerify=True, cache=None, *args, **kwargs): """ Input: dict BALLOTS: {int ballotid: (str imgpath_0, ...)} dict CACHE: {str imgpath: (tuple decodings, bool isflipped)} Output: (dict IMG2DECODING, dict FLIPMAP, dict BBSTRIPES_MAP, list ERR_IMGPATHS, list IOERR_IMGPATHS) """ topbot_paths = [[TOP_GUARD_IMGP, BOT_GUARD_IMGP], [TOP_GUARD_SKINNY_IMGP, BOT_GUARD_SKINNY_IMGP]] # DECODED_RESULTS_MAP: {ballotID: [(BCS, isflip, BBS, bbstripes_dict), ...]} decoded_results_map = decode_ballots(ballots, topbot_paths, manager, queue, skipVerify=skipVerify, cache=cache) flipmap = {} # maps {imgpath: bool isFlipped} bbstripes_map = {} # maps {'wideNarrow': [(str imgpath, (x1,y1,x2,y2), int id), ...], ...} err_imgpaths = [] ioerr_imgpaths = [] img2decoding = {} for ballotid, decoded_results in decoded_results_map.iteritems(): imgpaths = ballots[ballotid] for i, subtuple in enumerate(decoded_results): if issubclass(type(subtuple), IOError): ioerr_imgpaths.append(subtuple.filename) continue bcs, isflipped, bbs, bbstripes_dict = subtuple imgpath = imgpaths[i] flipmap[imgpath] = isflipped bc_ul = bcs[0] if not bc_ul: print "..error on: {0}".format(imgpath) err_imgpaths.append(imgpath) elif skipVerify: img2decoding[imgpath] = (bc_ul,) else: img2decoding[imgpath] = (bc_ul,) # First, maintain a global ordering of each stripe, for bbstripe_idx stripe_y1s = [] for label, bbstripes in bbstripes_dict.iteritems(): for (x1,y1,x2,y2) in bbstripes: stripe_y1s.append(y1) # sort by Y1, bottom->top stripe_y1s = sorted(stripe_y1s, key=lambda t: -t) for label, bbstripes in bbstripes_dict.iteritems(): for (x1,y1,x2,y2) in bbstripes: bbstripe_idx = stripe_y1s.index(y1) bbstripes_map.setdefault(label, []).append((imgpath, (x1,y1,x2,y2), bbstripe_idx)) def sizeit(thing): nbytes = asize.asizeof(thing) return "{0} bytes ({1} MB)".format(nbytes, nbytes / 1e6) total_bytes = asize.asizeof(decoded_results_map) + asize.asizeof(flipmap) + asize.asizeof(bbstripes_map) print "decoded_results_map: {0} ({1:.4f}%)".format(sizeit(decoded_results_map), 100.0 * (asize.asizeof(decoded_results_map) / float(total_bytes))) print "flipmap: {0} ({1:.4f}%)".format(sizeit(flipmap), 100.0 * (asize.asizeof(flipmap) / float(total_bytes))) print "bbstripes_map: {0} ({1:.4f}%)".format(sizeit(bbstripes_map), 100.0 * (asize.asizeof(bbstripes_map) / float(total_bytes))) print " Total: {0} bytes ({1} MB)".format(total_bytes, total_bytes / 1e6) total_bytes_oc = (666137 / (float(len(flipmap)))) * total_bytes print " Total (Extrapolate to Full OC): {0} MB ({1} GB)".format(total_bytes_oc / 1e6, total_bytes_oc / 1e9) """ TODO: Crazy Memory usage here. At this point, for full OC estimate, we would use: Total (Extrapolate to Full OC): 18955.6165387 MB (18.9556165387 GB) The huge hitters are decoded_results_map (~9 GB) and bbstripes_map (~11 GB). """ return img2decoding, flipmap, bbstripes_map, err_imgpaths, ioerr_imgpaths
def run(self): global GLOBAL_ID h, w = self.img1.shape bb = [0, h - 1, 0, w - 1] # list MATCHES: [(imgath, score1, score2, patch, i1, i2, j1, j2, # rszFac), ...] imgpatch = self.img1.astype('float32') / 255.0 if NPROCS == 1: print "(ThreadDoTempMatch): Using 1 process (TM_THRESHOLD={0}).".format( self.THRESHOLD) matches = shared.find_patch_matchesV1( imgpatch, bb[:], self.imgpaths, threshold=self.THRESHOLD, output_Ireg=False, save_patches=True, outdir=outdir, outid_start=GLOBAL_ID, jobid=jobid, progress_queue=progress_queue) else: print "(ThreadDoTempMatch): Using {0} processes. (TM_THRESHOLD={1})".format( NPROCS, self.THRESHOLD) procs = [] # [(Process p, int numimgs), ...] if progress_queue is not None: manager = multiprocessing.Manager() else: manager = progress_queue._manager queue = manager.Queue() step = int(math.ceil(len(self.imgpaths) / float(NPROCS))) for procnum in xrange(NPROCS): i_start = procnum * step i_end = i_start + step if procnum == NPROCS - 1: imgpaths_in = self.imgpaths[i_start:] else: imgpaths_in = self.imgpaths[i_start:i_end] outdir_in = os.path.join(outdir, "proc_{0}".format(procnum)) p = multiprocessing.Process(target=mp_patchmatch, args=(queue, imgpatch, bb[:], imgpaths_in), kwargs={ 'output_Ireg': False, 'save_patches': True, 'outdir': outdir_in, 'outid_start': GLOBAL_ID, 'jobid': jobid, 'progress_queue': progress_queue }) p.start() procs.append((p, len(imgpaths_in))) matches = [] for i, (proc, numimgs) in enumerate(procs): proc.join() p_matches = queue.get() print "(Process_{0}) Finished temp matching, got {1} matches ({2:.4f}s, {3} images total)".format( i, len(p_matches), dur, numimgs) matches.extend(p_matches) print "DONE with temp matching. Found: {0} matches ({1:.4f}s, {2:.5f}s per image)".format( len(matches), dur, dur / float(len(self.imgpaths))) GLOBAL_ID += len(matches) matches_size = asizeof(matches) print " sizeof(matches): {0} bytes ({1:.4f} MB)".format( matches_size, matches_size / 1e6) wx.CallAfter(self.callback, matches)
def decode_ballots(self, ballots, manager=None, queue=None, skipVerify=True, cache=None, *args, **kwargs): """ Input: dict BALLOTS: {int ballotid: (str imgpath_0, ...)} dict CACHE: {str imgpath: (tuple decodings, bool isflipped)} Output: (dict IMG2DECODING, dict FLIPMAP, dict BBSTRIPES_MAP, list ERR_IMGPATHS, list IOERR_IMGPATHS) """ topbot_paths = [[TOP_GUARD_IMGP, BOT_GUARD_IMGP], [TOP_GUARD_SKINNY_IMGP, BOT_GUARD_SKINNY_IMGP]] # DECODED_RESULTS_MAP: {ballotID: [(BCS, isflip, BBS, bbstripes_dict), # ...]} decoded_results_map = decode_ballots(ballots, topbot_paths, manager, queue, skipVerify=skipVerify, cache=cache) flipmap = {} # maps {imgpath: bool isFlipped} # maps {'wideNarrow': [(str imgpath, (x1,y1,x2,y2), int id), ...], ...} bbstripes_map = {} err_imgpaths = [] ioerr_imgpaths = [] img2decoding = {} for ballotid, decoded_results in decoded_results_map.iteritems(): imgpaths = ballots[ballotid] for i, subtuple in enumerate(decoded_results): if issubclass(type(subtuple), IOError): ioerr_imgpaths.append(subtuple.filename) continue bcs, isflipped, bbs, bbstripes_dict = subtuple imgpath = imgpaths[i] flipmap[imgpath] = isflipped bc_ul = bcs[0] if not bc_ul: error("error on: {0}", imgpath) err_imgpaths.append(imgpath) elif skipVerify: img2decoding[imgpath] = (bc_ul, ) else: img2decoding[imgpath] = (bc_ul, ) # First, maintain a global ordering of each stripe, for # bbstripe_idx stripe_y1s = [] for label, bbstripes in bbstripes_dict.iteritems(): for (x1, y1, x2, y2) in bbstripes: stripe_y1s.append(y1) # sort by Y1, bottom->top stripe_y1s = sorted(stripe_y1s, key=lambda t: -t) for label, bbstripes in bbstripes_dict.iteritems(): for (x1, y1, x2, y2) in bbstripes: bbstripe_idx = stripe_y1s.index(y1) bbstripes_map.setdefault(label, []).append( (imgpath, (x1, y1, x2, y2), bbstripe_idx)) def sizeit(thing): nbytes = asize.asizeof(thing) return "{0} bytes ({1} MB)".format(nbytes, nbytes / 1e6) total_bytes = asize.asizeof(decoded_results_map) + asize.asizeof( flipmap) + asize.asizeof(bbstripes_map) debug( "decoded_results_map: {0} ({1:.4f}%)", sizeit(decoded_results_map), 100.0 * (asize.asizeof(decoded_results_map) / float(total_bytes))) debug("flipmap: {0} ({1:.4f}%)", sizeit(flipmap), 100.0 * (asize.asizeof(flipmap) / float(total_bytes))) debug("bbstripes_map: {0} ({1:.4f}%)", sizeit(bbstripes_map), 100.0 * (asize.asizeof(bbstripes_map) / float(total_bytes))) debug(" Total: {0} bytes ({1} MB)", total_bytes, total_bytes / 1e6) total_bytes_oc = (666137 / (float(len(flipmap)))) * total_bytes debug(" Total (Extrapolate to Full OC): {0} MB ({1} GB)", total_bytes_oc / 1e6, total_bytes_oc / 1e9) """ TODO: Crazy Memory usage here. At this point, for full OC estimate, we would use: Total (Extrapolate to Full OC): 18955.6165387 MB (18.9556165387 GB) The huge hitters are decoded_results_map (~9 GB) and bbstripes_map (~11 GB). """ return img2decoding, flipmap, bbstripes_map, err_imgpaths, ioerr_imgpaths