def _eval(self): from tensorpack.utils.utils import get_tqdm_kwargs score = 0.0 ind = 0.0 x_valid, y_valid = self.data valid_predictions = [] with tqdm.tqdm(total=len(x_valid) // config.INFERENCE_BATCH + 1, **get_tqdm_kwargs()) as pbar: start = 0 end = 0 for i in range(len(x_valid) // config.INFERENCE_BATCH + 1): start = i * config.INFERENCE_BATCH end = start + config.INFERENCE_BATCH if start + config.INFERENCE_BATCH < len( x_valid) else len(x_valid) x = x_valid[start:end] final_probs, final_labels = self.pred(x) valid_predictions.extend(final_probs) #score += mapk(la, final_labels) pbar.update() valid_predictions = np.array(valid_predictions) map3 = mapk(self.valid_df[['y']].values, preds2catids(valid_predictions).values) print('Map3: {:.5f}'.format(map3)) self.trainer.monitors.put_scalar("Map3", map3)
def query_eval_output(df, pred_func, tqdm_bar=None): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] tqdm_bar: a tqdm object to be shared among multiple evaluation instances. If None, will create a new one. Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323 with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) for img, gt_boxes, gt_ids in df.get_data(): fvs = pred_func(img, gt_boxes) # print(fvs.shape) result_list = [] fv_list = [] id_list = [] for fv, gt_id in zip(fvs, gt_ids): fv_list.append(fv.tolist()) id_list.append(int(gt_id)) result_list.append(id_list) result_list.append(fv_list) all_results.append(result_list) tqdm_bar.update(1) return all_results
def pred_brats(df, detect_func): """ prediction """ df.reset_state() gts = [] results = [] with tqdm.tqdm(total=df.size(), **get_tqdm_kwargs()) as pbar: for filename, image_id, data in df.get_data(): final_label, probs = detect_func(data) if config.TEST_FLIP: pred_flip, probs_flip = detect_func(flip_lr(data)) final_prob = (probs + probs_flip) / 2.0 pred = np.argmax(final_prob, axis=-1) pred[pred == 3] = 4 if config.ADVANCE_POSTPROCESSING: pred = crop_ND_volume_with_bounding_box(pred, data['bbox'][0], data['bbox'][1]) pred = post_processing(pred, data['weights'][:,:,:,0]) pred = np.asarray(pred, np.int16) final_label = np.zeros(data['original_shape'], np.int16) final_label = set_ND_volume_roi_with_bounding_box_range(final_label, data['bbox'][0], data['bbox'][1], pred) else: final_label = pred save_to_nii(final_label, image_id, outdir="eval_out18", mode="label") # save prob to ensemble # save_to_pkl(probs, image_id, outdir="eval_out18_prob_{}".format(config.CROSS_VALIDATION)) pbar.update() return None
def eval_on_dataflow(df, detect_func): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns a dict Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] with tqdm.tqdm(total=df.size(), **get_tqdm_kwargs()) as pbar: for img, img_id in df.get_data(): results = detect_func(img) for classid, boxes, scores in results: cat_id = COCOMeta.class_id_to_category_id[classid] boxes[:, 2] -= boxes[:, 0] boxes[:, 3] -= boxes[:, 1] for box, score in zip(boxes, scores): all_results.append({ 'image_id': img_id, 'category_id': cat_id, 'bbox': list(map(lambda x: float(round(x, 1)), box)), 'score': float(round(score, 2)), }) pbar.update(1) return all_results
def _eval(self): from tensorpack.utils.utils import get_tqdm_kwargs valid_predictions = [] valid_y = [] th = 0.15 with tqdm.tqdm(total=len(self.valid_ds) // config.INFERENCE_BATCH + 1, **get_tqdm_kwargs()) as pbar: for i in range(len(self.valid_ds) // config.INFERENCE_BATCH + 1): start = i * config.INFERENCE_BATCH end = start + config.INFERENCE_BATCH if start + config.INFERENCE_BATCH < len( self.valid_ds) else len(self.valid_ds) data = self.valid_ds[start:end] data = [preprocess(d, is_training=False) for d in data] x = np.array([_[0] for _ in data]) y = np.array([_[1] for _ in data]) final_probs = self.pred(x) valid_predictions.extend(np.argmax(final_probs[0], axis=-1)) valid_y.extend(y) pbar.update() valid_predictions = np.array(valid_predictions) valid_y = np.array(valid_y) score = (valid_y == valid_predictions).mean() self.trainer.monitors.put_scalar("score", score) return score
def eval_with_funcs(predictors, nr_eval, get_player_fn, verbose=False): """ Args: predictors ([PredictorBase]) """ class Worker(StoppableThread, ShareSessionThread): def __init__(self, func, func_refine, queue): super(Worker, self).__init__() self._func = func self._func_refine = func_refine self.q = queue def func(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func(*args, **kwargs) def func_refine(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func_refine(*args, **kwargs) def run(self): with self.default_sess(): player = get_player_fn() while not self.stopped(): try: r, tp = play_one_episode(player, self.func, self.func_refine, verbose) except RuntimeError: return self.queue_put_stoppable(self.q, (r, tp)) q = queue.Queue() threads = [Worker(f[0], f[1], q) for f in predictors] for k in threads: k.start() time.sleep(0.1) # avoid simulator bugs stat_r = StatCounter() stat_tp = StatCounter() def fetch(): r, tp = q.get() stat_r.feed(r) stat_tp.feed(tp) if verbose: logger.info("reward %f" % r) logger.info("true positive %f" % tp) for _ in tqdm(range(nr_eval), **get_tqdm_kwargs()): fetch() logger.info("Waiting for all the workers to finish the last run...") for k in threads: k.stop() for k in threads: k.join() while q.qsize(): fetch() return stat_r.average, stat_tp.average
def eval_on_dataflow(df, detect_func, tqdm_bar=None): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] tqdm_bar: a tqdm object to be shared among multiple evaluation instances. If None, will create a new one. Return: list of dict, to be dumped to COCO json format """ csv_path = os.path.join(config.BASEDIR, 'train_ship_segmentations_v2.csv') seg_df = pd.read_csv(csv_path, engine="python") seg_df = seg_df.dropna(axis=0) seg_df = seg_df.set_index('ImageId') df.reset_state() all_results = [] # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323 with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) score = 0.0 all_score = [] count = 0.0 eval_names = [] eval_imgs = [] all_det = [] all_im = [] for img, img_id in df.get_data(): results = detect_func(img) #mask_whole = detect_func(img) #all_det.append(mask_whole) all_im.append(img) eval_names.append(img_id) final = draw_final_outputs(img, results) cv2.imwrite('./eval_out_bb/{}'.format(img_id), final) mask_instances = [r.mask for r in results] score_instances = [r.score for r in results] masks = clean_overlap_instance(mask_instances, score_instances, img_id) if len(masks) == 0: print("no mask!!", img_id) v = 0 else: v = local_eval(masks, img_id, seg_df) #pred, imgId score += v all_score.append(v) count += 1 tqdm_bar.update(1) for k in np.array(all_score).argsort()[:20]: print(all_score[k], eval_names[k]) # cv2.imwrite("./eval_out/{}".format(eval_names[k]), all_im[k]) # cv2.imwrite("./eval_out/{}_mask.jpg".format(eval_names[k].split(".")[0]), all_det[k]*255) print("Local Eval: ", score / count) return all_results, score / count
def main_loop(self): # some final operations that might modify the graph logger.info("[{}] Initializing graph variables ...".format(os.environ['PBS_ARRAY_INDEX'])) #self.sess.run(tf.initialize_all_variables()) self.config.session_init.init(self.sess) # tf.get_default_graph().finalize() callbacks = self.config.callbacks logger.info("[{}] Starting concurrency...".format(os.environ['PBS_ARRAY_INDEX'])) self._start_concurrency() #with self.sess.as_default(): logger.info("[{}] Setting default session".format(os.environ['PBS_ARRAY_INDEX'])) with ops.default_session(self.sess): try: logger.info("[{}] Getting global step".format(os.environ['PBS_ARRAY_INDEX'])) self.global_step = get_global_step() logger.info("[{}] Start training with global_step={}".format(os.environ['PBS_ARRAY_INDEX'], self.global_step)) if self.config.extra_arg['is_chief']: server = neptune_mp_server.Server( self.config.extra_arg['n_workers'], port=self.config.extra_arg['port'], debug_charts=self.config.extra_arg['debug_charts'], adam_debug=self.config.extra_arg['adam_debug'], schedule_hyper=self.config.extra_arg['schedule_hyper'], experiment_dir=self.config.extra_arg['experiment_dir']) server.main_loop() callbacks.before_train() for epoch in range(self.config.starting_epoch, self.config.max_epoch+1): with timed_operation( 'Epoch {}, global_step={}'.format( epoch, self.global_step + self.config.step_per_epoch)): for step in tqdm.trange( self.config.step_per_epoch, **get_tqdm_kwargs(leave=True)): if self.coord.should_stop(): return self.run_step() callbacks.trigger_step() try: self.global_step += 1 except: self.global_step = -1 self.trigger_epoch() print 'EPOCH ENDS HERE' except (KeyboardInterrupt, Exception): raise finally: # Do I need to run queue.close? print('Handling finally block') callbacks.after_train() self.coord.request_stop() self.summary_writer.close() self.sess.close()
def eval_with_funcs(predictors, nr_eval, get_player_fn, verbose=False): """ Args: predictors ([PredictorBase]) """ class Worker(StoppableThread, ShareSessionThread): def __init__(self, func, queue): super(Worker, self).__init__() self._func = func self.q = queue def func(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func(*args, **kwargs) def run(self): with self.default_sess(): player = get_player_fn() while not self.stopped(): try: val = play_one_episode(player, self.func) except RuntimeError: return self.queue_put_stoppable(self.q, val) q = queue.Queue() threads = [Worker(f, q) for f in predictors] for k in threads: k.start() time.sleep(0.1) # avoid simulator bugs stat = StatCounter() def fetch(): val = q.get() stat.feed(val) if verbose: if val > 0: logger.info("farmer wins") else: logger.info("lord wins") for _ in tqdm(range(nr_eval), **get_tqdm_kwargs()): fetch() logger.info("Waiting for all the workers to finish the last run...") for k in threads: k.stop() for k in threads: k.join() while q.qsize(): fetch() farmer_win_rate = stat.average return farmer_win_rate
def eval_with_funcs(predictors, nr_eval, get_player_fn, verbose=False): """ Args: predictors ([PredictorBase]) """ class Worker(StoppableThread, ShareSessionThread): def __init__(self, func, queue): super(Worker, self).__init__() self._func = func self.q = queue def func(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func(*args, **kwargs) def run(self): with self.default_sess(): player = get_player_fn(train=False) while not self.stopped(): try: score = play_one_episode(player, self.func) except RuntimeError: return self.queue_put_stoppable(self.q, score) q = queue.Queue() threads = [Worker(f, q) for f in predictors] for k in threads: k.start() time.sleep(0.1) # avoid simulator bugs stat = StatCounter() def fetch(): r = q.get() stat.feed(r) if verbose: logger.info("Score: {}".format(r)) for _ in tqdm(range(nr_eval), **get_tqdm_kwargs()): fetch() # waiting is necessary, otherwise the estimated mean score is biased logger.info("Waiting for all the workers to finish the last run...") for k in threads: k.stop() for k in threads: k.join() while q.qsize(): fetch() if stat.count > 0: return (stat.average, stat.max) return (0, 0)
def eval_with_funcs(predictors, nr_eval, get_player_fn, verbose=False): """ Args: predictors ([PredictorBase]) """ class Worker(StoppableThread, ShareSessionThread): def __init__(self, func, queue): super(Worker, self).__init__() self._func = func self.q = queue def func(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func(*args, **kwargs) def run(self): with self.default_sess(): player = get_player_fn(train=False) while not self.stopped(): try: score = play_one_episode(player, self.func) except RuntimeError: return self.queue_put_stoppable(self.q, score) q = queue.Queue() threads = [Worker(f, q) for f in predictors] for k in threads: k.start() time.sleep(0.1) # avoid simulator bugs stat = StatCounter() def fetch(): r = q.get() stat.feed(r) if verbose: logger.info("Score: {}".format(r)) for _ in tqdm(range(nr_eval), **get_tqdm_kwargs()): fetch() # waiting is necessary, otherwise the estimated mean score is biased logger.info("Waiting for all the workers to finish the last run...") for k in threads: k.stop() for k in threads: k.join() while q.qsize(): fetch() if stat.count > 0: return (stat.average, stat.max) return (0, 0)
def _eval(self): from tensorpack.utils.utils import get_tqdm_kwargs score = 0.0 ind = 0.0 self.df.reset_state() with tqdm.tqdm(total=self.df.size(), **get_tqdm_kwargs()) as pbar: for img, la in self.df.get_data(): ind += 1 final_probs, final_labels = self.pred(img) score += (np.array(final_labels) == np.array(la)).mean() pbar.update() print("Val Acc: ", score / ind) self.trainer.monitors.put_scalar("Acc", score / ind)
def re_id_eval(args): with open(args.gallery_file, 'r') as gallery_file: gallery_list = json.load(gallery_file) gallery_bb = [] for frame in gallery_list: """ frame[0]: abs fname frame[1]: bb list frame[2]: label list frame[3]: score list frame[4]: feature vectors list """ viz_detection(args, frame[0], frame[1]) gt_bb_array, gt_cls_array = read_annotations( os.path.join(args.anno_dir, os.path.basename(frame[0]).split('.')[0] + '.txt')) if not frame[1]: # print('No detection') continue det_gt_cls_array, pos_ind = bb_cls_matching( np.array(frame[1]), gt_bb_array, gt_cls_array) for bb, fv, det_cls in zip(frame[1], frame[4], det_gt_cls_array): gallery_bb.append(fv + bb + [det_cls]) gallery_bb = np.array(gallery_bb) print(gallery_bb.shape) with open(args.query_file, 'r') as query_file: query_list = json.load(query_file) tp_top20 = 0.0 with tqdm.tqdm(total=len(query_list), **get_tqdm_kwargs()) as tqdm_bar: for query in query_list: """ query[0]: feature vector list query[1]: query id list """ fv = np.array(query[1][0][0]) distance = [] for gfv in gallery_bb: distance.append(np.linalg.norm(fv - gfv[:256])) distance_array = np.array(distance) index_sort = np.argsort(distance_array) cls_top20 = gallery_bb[index_sort[:20], 260] if query[0][0] in cls_top20.astype(int).tolist(): # print(query[0][0], cls_top20.astype(int).tolist()) print('yay') tp_top20 += 1 tqdm_bar.update(1) print('Top 20 accuracy: ' + str(tp_top20 / len(query_list)))
def classifier_eval_output(df, pred_func, tqdm_bar=None): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] tqdm_bar: a tqdm object to be shared among multiple evaluation instances. If None, will create a new one. Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] jsonable = False # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323 with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) for fname, img, orig_shape in df.get_data(): bbs, probs = pred_func(img, orig_shape) if VIZ: input_file = os.path.join( '/media/yingges/TOSHIBA EXT/datasets/re-ID/PRW-v16.04.20/frames', os.path.basename(fname).split('.')[0] + '.jpg') img = cv2.imread(input_file, cv2.IMREAD_COLOR) final = draw_final_outputs(img, bbs, tags_on=False, bb_list_input=True) viz = np.concatenate((img, final), axis=1) cv2.imwrite(os.path.basename(input_file), viz) result_list = [] result_list.append(fname) bb_list = [] prob_list = [] for bb, prob in zip(bbs, probs): bb_list.append(list(map(lambda x: round(float(x), 2), bb))) prob_list.append(list(map(lambda x: round(float(x), 4), prob))) result_list.append(bb_list) result_list.append(prob_list) if not jsonable: jsonable = jsonable_test(result_list) all_results.append(result_list) tqdm_bar.update(1) return all_results
def start(self): """ Start testing with a progress bar. """ itr = self.ds.__iter__() if self.warmup: for _ in tqdm.trange(self.warmup, **get_tqdm_kwargs()): next(itr) # add smoothing for speed benchmark with get_tqdm(total=self.test_size, leave=True, smoothing=0.2) as pbar: for idx, dp in enumerate(itr): pbar.update() if idx == self.test_size - 1: break
def eval_with_funcs(predictors, nr_eval, get_player_fn): class Worker(StoppableThread, ShareSessionThread): def __init__(self, func, queue): super(Worker, self).__init__() self._func = func self.q = queue def func(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func(*args, **kwargs) def run(self): with self.default_sess(): player = get_player_fn(train=False) while not self.stopped(): try: score = play_one_episode(player, self.func) # print("Score, ", score) except RuntimeError: return self.queue_put_stoppable(self.q, score) q = queue.Queue() threads = [Worker(f, q) for f in predictors] for k in threads: k.start() time.sleep(0.1) # avoid simulator bugs stat = StatCounter() try: for _ in tqdm(range(nr_eval), **get_tqdm_kwargs()): r = q.get() stat.feed(r) logger.info("Waiting for all the workers to finish the last run...") for k in threads: k.stop() for k in threads: k.join() while q.qsize(): r = q.get() stat.feed(r) except: logger.exception("Eval") finally: if stat.count > 0: return (stat.average, stat.max) return (0, 0)
def eval_brats(df, detect_func, with_gt=True): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] Returns: list of dict, to be dumped to COCO json format """ df.reset_state() gts = [] results = [] with tqdm.tqdm(total=df.size(), **get_tqdm_kwargs()) as pbar: for filename, image_id, data in df.get_data(): final_label, probs = detect_func(data) if config.TEST_FLIP: pred_flip, probs_flip = detect_func(flip_lr(data)) final_prob = (probs + probs_flip) / 2.0 pred = np.argmax(final_prob, axis=-1) pred[pred == 3] = 4 if config.ADVANCE_POSTPROCESSING: pred = crop_ND_volume_with_bounding_box( pred, data['bbox'][0], data['bbox'][1]) pred = post_processing(pred, data['weights'][:, :, :, 0]) pred = np.asarray(pred, np.int16) final_label = np.zeros(data['original_shape'], np.int16) final_label = set_ND_volume_roi_with_bounding_box_range( final_label, data['bbox'][0], data['bbox'][1], pred) else: final_label = pred gt = load_nifty_volume_as_array("{}/{}_seg.nii.gz".format( filename, image_id)) gts.append(gt) results.append(final_label) pbar.update() test_types = ['whole', 'core', 'enhancing'] ret = {} for type_idx in range(3): dice = dice_of_brats_data_set(gts, results, type_idx) dice = np.asarray(dice) dice_mean = dice.mean(axis=0) dice_std = dice.std(axis=0) test_type = test_types[type_idx] ret[test_type] = dice_mean[0] print('tissue type', test_type) print('dice mean', dice_mean) return ret
def eval_coco(df, detect_func, tqdm_bar=None): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] tqdm_bar: a tqdm object to be shared among multiple evaluation instances. If None, will create a new one. Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323 with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) for img, img_id in df.get_data(): results = detect_func(img) for r in results: box = r.box cat_id = mapper.get( r.class_id, 'background' ) #COCOMeta.class_id_to_category_id[r.class_id] # this two lines are coco format(xywh) # ignore # box[2] -= box[0] # box[3] -= box[1] res = { 'image_id': img_id, 'category_id': cat_id, 'bbox': list(map(lambda x: round(float(x), 2), box)), 'score': round(float(r.score), 3), } # also append segmentation to results if r.mask is not None: rle = cocomask.encode( np.array(r.mask[:, :, None], order='F'))[0] rle['counts'] = rle['counts'].decode('ascii') res['segmentation'] = rle all_results.append(res) tqdm_bar.update(1) return all_results
def eval_output(df, detect_func, tqdm_bar=None): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] tqdm_bar: a tqdm object to be shared among multiple evaluation instances. If None, will create a new one. Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323 with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) for img, img_fname in df.get_data(): results = detect_func(img) result_list = [] result_list.append(img_fname) bb_list = [] label_list = [] score_list = [] fv_list = [] for r in results: box = r.box bb_list.append(list(map(lambda x: round(float(x), 2), box))) label_list.append(int(r.class_id)) score_list.append(round(float(r.score), 3)) # print(len(r.fv.tolist())) fv_list.append(r.fv.tolist()) result_list.append(bb_list) result_list.append(label_list) result_list.append(score_list) result_list.append(fv_list) # dump a dummy json here to check for validity all_results.append(result_list) tqdm_bar.update(1) return all_results
def main_loop(self): # some final operations that might modify the graph self._init_summary() get_global_step_var( ) # ensure there is such var, before finalizing the graph logger.info("Setup callbacks ...") callbacks = self.config.callbacks callbacks.setup_graph(self) # TODO use weakref instead? logger.info("Initializing graph variables ...") self.sess.run(tf.initialize_all_variables()) self.config.session_init.init(self.sess) tf.get_default_graph().finalize() self._start_concurrency() with self.sess.as_default(): try: self.global_step = get_global_step() logger.info("Start training with global_step={}".format( self.global_step)) callbacks.before_train() for epoch in range(self.config.starting_epoch, self.config.max_epoch + 1): with timed_operation('Epoch {}, global_step={}'.format( epoch, self.global_step + self.config.step_per_epoch)): for step in tqdm.trange(self.config.step_per_epoch, **get_tqdm_kwargs(leave=True)): if self.coord.should_stop(): return self.run_step() #callbacks.trigger_step() # not useful? self.global_step += 1 self.trigger_epoch() except (KeyboardInterrupt, Exception): raise finally: # Do I need to run queue.close? callbacks.after_train() self.coord.request_stop() self.summary_writer.close() self.sess.close()
def eval_coco(df, detect_func, tqdm_bar=None): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] tqdm_bar: a tqdm object to be shared among multiple evaluation instances. If None, will create a new one. Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323 with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) for img, img_id in df: results = detect_func(img) for r in results: box = r.box cat_id = COCOMeta.class_id_to_category_id[r.class_id] box[2] -= box[0] box[3] -= box[1] res = { 'image_id': img_id, 'category_id': cat_id, 'bbox': list(map(lambda x: round(float(x), 3), box)), 'score': round(float(r.score), 4), } # also append segmentation to results if r.mask is not None: rle = cocomask.encode( np.array(r.mask[:, :, None], order='F'))[0] rle['counts'] = rle['counts'].decode('ascii') res['segmentation'] = rle all_results.append(res) tqdm_bar.update(1) return all_results
def predict_dataflow(df, model_func, tqdm_bar=None): """ Args: df: a DataFlow which produces (image, image_id) model_func: a callable from the TF model. It takes image and returns (boxes, probs, labels, [masks]) tqdm_bar: a tqdm object to be shared among multiple evaluation instances. If None, will create a new one. Returns: list of dict, in the format used by `DetectionDataset.eval_or_save_inference_results` """ df.reset_state() all_results = [] # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323 with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) for img, img_id in df: results = predict_image(img, model_func) for r in results: res = { 'image_id': img_id, 'category_id': r.class_id, 'bbox': list(r.box), 'score': round(float(r.score), 4), } # also append segmentation to results if r.mask is not None: rle = cocomask.encode( np.array(r.mask[:, :, None], order='F'))[0] rle['counts'] = rle['counts'].decode('ascii') res['segmentation'] = rle all_results.append(res) tqdm_bar.update(1) return all_results
def _eval(self): from tensorpack.utils.utils import get_tqdm_kwargs valid_predictions = [] valid_y = [] valid_logits = [] th = 0.15 total_run = len(self.valid_ds) // config.INFERENCE_BATCH total_run = total_run + 1 if len( self.valid_ds) % config.INFERENCE_BATCH != 0 else total_run with tqdm.tqdm(total=total_run, **get_tqdm_kwargs()) as pbar: for i in range(total_run): start = i * config.INFERENCE_BATCH end = start + config.INFERENCE_BATCH if start + config.INFERENCE_BATCH < len( self.valid_ds) else len(self.valid_ds) data = self.valid_ds[start:end] data = [preprocess(d, is_training=False) for d in data] x = np.array([_[0] for _ in data]) y = np.array([_[1] for _ in data]) if len(x) == 0: break final_probs = self.pred(x) valid_predictions.extend(final_probs[0]) valid_logits.extend(final_probs[0]) valid_y.extend(y) #score += mapk(la, final_labels) pbar.update() valid_predictions = np.array(valid_predictions) valid_y = np.array(valid_y) valid_logits = np.array(valid_logits) val_loss = tf_ce(valid_logits, valid_y) F1_score_05 = calc_macro_f1(valid_predictions, valid_y, 0.5) F1_score_015 = calc_macro_f1(valid_predictions, valid_y, 0.15) F1_score_02 = calc_macro_f1(valid_predictions, valid_y, 0.2) print('F1_score: {:.5f} {:.5f} {:.5f}'.format(F1_score_05, F1_score_015, F1_score_02)) self.trainer.monitors.put_scalar("F1_score", F1_score_015) return F1_score_015
def eval_on_dataflow(df, detect_func): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] with tqdm.tqdm(total=df.size(), **get_tqdm_kwargs()) as pbar: for img, img_id in df.get_data(): results = detect_func(img) for r in results: box = r.box #cat_id = COCOMeta.class_id_to_category_id[r.class_id] cat_id = r.class_id box[2] -= box[0] box[3] -= box[1] res = { 'image_id': img_id, 'category_id': cat_id, 'bbox': list(map(lambda x: float(round(x, 1)), box)), 'score': float(round(r.score, 2)), } # also append segmentation to results if r.mask is not None: # rle = cocomask.encode( # np.array(r.mask[:, :, None], order='F'))[0] # rle['counts'] = rle['counts'].decode('ascii') res['segmentation'] = r.mask all_results.append(res) pbar.update(1) return all_results
def eval_coco(df, detect_func): """ Args: df: a DataFlow which produces (image, image_id) detect_func: a callable, takes [image] and returns [DetectionResult] Returns: list of dict, to be dumped to COCO json format """ df.reset_state() all_results = [] with tqdm.tqdm(total=df.size(), **get_tqdm_kwargs()) as pbar: for img, img_id in df.get_data(): results = detect_func(img) for r in results: box = r.box cat_id = COCOMeta.class_id_to_category_id[r.class_id] box[2] -= box[0] box[3] -= box[1] res = { 'image_id': img_id, 'category_id': cat_id, 'bbox': list(map(lambda x: float(round(x, 1)), box)), 'score': float(round(r.score, 2)), } # also append segmentation to results if r.mask is not None: rle = cocomask.encode( np.array(r.mask[:, :, None], order='F'))[0] rle['counts'] = rle['counts'].decode('ascii') res['segmentation'] = rle all_results.append(res) pbar.update(1) return all_results
def eval_W(df, eval_one_image, tqdm_bar=None): df.reset_state() all_results = [] with ExitStack() as stack: if tqdm_bar is None: tqdm_bar = stack.enter_context( tqdm.tqdm(total=df.size(), **get_tqdm_kwargs())) for img, boxes, img_id, male, longhair, sunglass, hat, tshirt, longsleeve, \ formal, shorts, jeans, skirt, facemask, logo, stripe, longpants in df: results = eval_one_image(img, boxes, male, longhair, sunglass, hat, tshirt, longsleeve, formal, shorts, jeans, skirt, facemask, logo, stripe, longpants) male, longhair, sunglass, hat, tshirt, longsleeve, \ formal, shorts, jeans, skirt, facemask, logo, stripe, longpants, \ male_predict, longhair_predict, sunglass_predict, hat_predict, tshirt_predict, \ longsleeve_predict, formal_predict, shorts_predict, jeans_predict, skirt_predict, \ facemask_predict, logo_predict, stripe_predict, longpants_predict = list( zip(*[[int(r.male), int(r.longhair), int(r.sunglass), int(r.hat), int(r.tshirt), int(r.longsleeve), int(r.formal), int(r.shorts), int(r.jeans), int(r.skirt), int(r.facemask), int(r.logo), int(r.stripe), int(r.longpants), float(r.male_predict), float(r.longhair_predict), float(r.sunglass_predict), float(r.hat_predict), float(r.tshirt_predict), float(r.longsleeve_predict), float(r.formal_predict), float(r.shorts_predict), float(r.jeans_predict), float(r.skirt_predict), float(r.facemask_predict), float(r.logo_predict), float(r.stripe_predict), float(r.longpants_predict)] for r in results])) res = { 'image_id': int(img_id), 'male': male, 'longhair': longhair, 'sunglass': sunglass, 'hat': hat, 'tshirt': tshirt, 'longsleeve': longsleeve, 'formal': formal, 'shorts': shorts, 'jeans': jeans, 'skirt': skirt, 'facemask': facemask, 'logo': logo, 'stripe': stripe, 'longpants': longpants, 'male_predict': male_predict, 'longhair_predict': longhair_predict, 'sunglass_predict': sunglass_predict, 'hat_predict': hat_predict, 'tshirt_predict': tshirt_predict, 'longsleeve_predict': longsleeve_predict, 'formal_predict': formal_predict, 'shorts_predict': shorts_predict, 'jeans_predict': jeans_predict, 'skirt_predict': skirt_predict, 'facemask_predict': facemask_predict, 'logo_predict': logo_predict, 'stripe_predict': stripe_predict, 'longpants_predict': longpants_predict } all_results.append(res) tqdm_bar.update(1) return all_results
def test_rpcio(): from . import dataio_recv, dataio_send import sys, os # if len(sys.argv) > 2: # # client mode # pass # else: from .server import init_rpcio_server host = '127.0.0.1' port = 50000 dataio = init_rpcio_server(host, port) batchSize = 4096 subBatchSize = 256 clients = [] with tf.Session() as sess: #, tf.device(':/cpu:0'): dataios = [] import multiprocessing as mp dim0 = 256 dim1 = 512 for qidx in range(4): v0, v1 = dataio_recv("main%d" % qidx, batchSize, [tf.float32, tf.int32], [[dim0], [dim1]], sub_processor_batch_size=subBatchSize) op_send = dataio_send([v0 * 2, v1 * 2], 'main%d' % qidx, batchSize, sub_processor_batch_size=subBatchSize) for idx in range(batchSize // subBatchSize): proc = mp.Process(target=_client_loop, args=('main%d' % qidx, idx, subBatchSize, dim0, dim1)) clients.append(proc) proc.start() dataios.append(op_send) sess.run(tf.global_variables_initializer()) def thread_run_op(op, qidx): print("thread {} start ".format(qidx)) for _ in range(100000): sess.run(op) for qidx, op in enumerate(dataios): if qidx == 0: continue import threading threading._start_new_thread(thread_run_op, (op, qidx)) from tensorpack.utils.utils import get_tqdm_kwargs import tqdm for _ in tqdm.tqdm(range(100000), **get_tqdm_kwargs()): try: sess.run(dataios[0]) except KeyboardInterrupt: sess.close() break from time import sleep # sleep(1) dataio.close() pass
def eval_with_funcs(predictors, nr_eval, get_player_fn, directory=None, files_list=None): """ Args: predictors ([PredictorBase]) Runs episodes in parallel, returning statistics about the model performance. """ class Worker(StoppableThread, ShareSessionThread): def __init__(self, func, queue, distErrorQueue): super(Worker, self).__init__() self._func = func self.q = queue self.q_dist = distErrorQueue def func(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func(*args, **kwargs) def run(self): with self.default_sess(): player = get_player_fn(directory=directory, task=False, files_list=files_list) while not self.stopped(): try: score, filename, ditance_error, q_values = play_one_episode( player, self.func) # print("Score, ", score) except RuntimeError: return self.queue_put_stoppable(self.q, score) self.queue_put_stoppable(self.q_dist, ditance_error) q = queue.Queue() q_dist = queue.Queue() threads = [Worker(f, q, q_dist) for f in predictors] # start all workers for k in threads: k.start() time.sleep(0.1) # avoid simulator bugs stat = StatCounter() dist_stat = StatCounter() # show progress bar w/ tqdm for _ in tqdm(range(nr_eval), **get_tqdm_kwargs()): r = q.get() stat.feed(r) dist = q_dist.get() dist_stat.feed(dist) logger.info("Waiting for all the workers to finish the last run...") for k in threads: k.stop() for k in threads: k.join() while q.qsize(): r = q.get() stat.feed(r) while q_dist.qsize(): dist = q_dist.get() dist_stat.feed(dist) if stat.count > 0: return (stat.average, stat.max, dist_stat.average, dist_stat.max) return (0, 0, 0, 0)
def eval_with_funcs(predictors, nr_eval, get_player_fn, verbose=False): """ Args: predictors ([PredictorBase]) """ class Worker(StoppableThread, ShareSessionThread): def __init__(self, func, queue): super(Worker, self).__init__() self._func = func self.q = queue def func(self, *args, **kwargs): if self.stopped(): raise RuntimeError("stopped!") return self._func(*args, **kwargs) def run(self): with self.default_sess(): player = get_player_fn() while not self.stopped(): try: stats = play_one_episode(player, self.func) except RuntimeError: return scores = [ stat.average if stat.count > 0 else -1 for stat in stats ] self.queue_put_stoppable(self.q, scores) q = queue.Queue() threads = [Worker(f, q) for f in predictors] for k in threads: k.start() time.sleep(0.1) # avoid simulator bugs stats = [StatCounter() for _ in range(7)] def fetch(): scores = q.get() for i, score in enumerate(scores): if scores[i] >= 0: stats[i].feed(scores[i]) accs = [stat.average if stat.count > 0 else 0 for stat in stats] if verbose: logger.info("passive decision accuracy: {}\n" "passive bomb accuracy: {}\n" "passive response accuracy: {}\n" "active decision accuracy: {}\n" "active response accuracy: {}\n" "active sequence accuracy: {}\n" "minor response accuracy: {}\n".format( accs[0], accs[1], accs[2], accs[3], accs[4], accs[5], accs[6])) for _ in tqdm(range(nr_eval), **get_tqdm_kwargs()): fetch() logger.info("Waiting for all the workers to finish the last run...") for k in threads: k.stop() for k in threads: k.join() while q.qsize(): fetch() accs = [stat.average if stat.count > 0 else 0 for stat in stats] return accs
PredictConfig( model=ResnetModel(), session_init=get_model_loader(args.load), input_names=['image'], output_names=get_resnet_model_output_names())) from tensorpack.utils.utils import get_tqdm_kwargs valid_ds = get_dataflow(is_train=False) valid_predictions = [] valid_y = [] valid_logits = [] th = 0.5 total_run = len(valid_ds) // config.INFERENCE_BATCH total_run = total_run + 1 if len( valid_ds) % config.INFERENCE_BATCH != 0 else total_run with tqdm.tqdm(total=total_run, **get_tqdm_kwargs()) as pbar: for i in range(total_run): start = i * config.INFERENCE_BATCH end = start + config.INFERENCE_BATCH if start + config.INFERENCE_BATCH < len( valid_ds) else len(valid_ds) data = valid_ds[start:end] data = [preprocess(d, is_training=False) for d in data] x = np.array([_[0] for _ in data]) y = np.array([_[1] for _ in data]) final_probs = pred(x) valid_predictions.extend(final_probs[0]) valid_logits.extend(final_probs[0]) valid_y.extend(y) pbar.update() valid_predictions = np.array(valid_predictions)
model=ResnetModel(), session_init=get_model_loader(args.load), input_names=['image'], output_names=get_resnet_model_output_names())) images = ResnetDetection.load_many(config.BASEDIR, config.VAL_DATASET) images = list(images) imgs = [img['image_data'] for img in images] labels = [img['with_ship'] for img in images] from tensorpack.utils.utils import get_tqdm_kwargs score = 0.0 ind = 0.0 b_size = config.RESNET_BATCH final_pred_mask = [] with tqdm.tqdm(total=len(imgs) // b_size + 1, **get_tqdm_kwargs()) as pbar: for i in range(len(imgs) // b_size + 1): ind += 1 start = i * b_size end = start + b_size if start + b_size < len( imgs) else len(imgs) batch_image = imgs[start:end] batch_image = np.array([ cv2.resize( cv2.imread(im), (config.RESNET_SIZE, config.RESNET_SIZE)) for im in batch_image ]) batch_label = np.array(labels[start:end]) final_probs, final_labels = pred(batch_image) final_pred_mask += list(final_labels)