def test_load_state(self): dt = DummyTrainer(6, 2) dd1, dd2 = DummyDataset(), DummyDataset() with suppress_stdout(): dt.fit(dd1, dd2) dt.test(dd1) dt = DummyTrainer(6, 2, checkpoint_dir='tests/ex', save_frequency=3) with suppress_stdout(): dt.fit(dd1, dd2) dt.test(dd1, 'epoch_3') shutil.rmtree('tests/ex')
def interactive(self, **kwargs): with suppress_stdout(): args = self._default_args(**kwargs) ans = interactive(args) print(ans)
def speech2txt(buff, trigger): resource_path = Path(os.getcwd()) / 'resources' duration = 2 fs = 16000 nsample = duration * fs nclass = 2 try: trigger.set() with utils.suppress_stdout(): log_speech.info("loading speech2txt model") m = model.full_model(fs, nsample, nclass=nclass, n_hop=128, load_model=True, resource_path=resource_path) while trigger.wait(): log_speech.info("speech prediction is triggered") print("choose right or left after the beep") time.sleep(1) audio = record_audio.record(duration, fs, resource_path) pred = 'r' if predict.predict(m, audio) else 'l' buff.put(pred, block=True) log_speech.info('model predicted %s' % pred) trigger.clear() except TypeError as e: log_speech.error('Failed to import utils, audio, or nn') log_speech.error(str(e)) except KeyboardInterrupt as e: log_speech.info("Forcebly closed by user: %s" % str(e))
def load_block(block_file): with open(block_file) as infile: with utils.suppress_stdout(): # data_block, numpy array of shape: [B x 3 x H x W] data_block = hickle.load(infile) # data_block, numpy array of shape: [B x H x W x 3] data_block = np.transpose(data_block, (0, 2, 3, 1)) # to HWC return data_block
def __mape_data_detailed_test_split(self, seed): if seed % 5 == 0: print("Test iteration {} of {}".format(seed + 1, self.number_tests)) self.train, self.test = self.test_splitting.get_split(seed, self.model.data) # Remove duped rows (possibly duplication has been provided) self.test = self.test.drop('cluster', axis=1).drop_duplicates() # Apply account test filter if not self.account_test_filter is None: self.test = self.test[self.test['account_banner'].isin(self.account_test_filter)] self.train_x = self.train.loc[:, self.train.columns != self.model.target] self.train_y = self.train.loc[:, self.train.columns == self.model.target] self.test_x = self.test.loc[:, self.test.columns != self.model.target] # test_x['baseline_units_original'] = test_x['baseline_units'] # test_x['baseline_units'] = test_x['baseline_units_2'] self.test_y = self.test.loc[:, self.test.columns == self.model.target] # transform to Series self.train_y = self.train_y.squeeze() self.test_y = self.test_y.squeeze() # Train model and validate performance with suppress_stdout(): self.model.model.fit(self.train_x, self.train_y) self.pred_train_y = self.model.model.predict(self.train_x) self.pred_test_y = self.model.model.predict(self.test_x) m = dict() for account in self.test['account_banner'].unique(): m_acc = dict() for product_25 in self.test[self.test['account_banner']==account]['original_product_dimension_25'].unique(): m_25 = dict() for product_26 in self.test[(self.test['account_banner']==account) & \ (self.test['original_product_dimension_25']==product_25)] \ ['original_product_dimension_26'].unique(): m_26 = dict() metric_dict = self.get_metrics_by_account_product25_product26(['mape_m', 'mape_h'], account, product_25, product_26) m_26['mape_m'] = metric_dict['mape_m'] m_26['mape_h'] = metric_dict['mape_h'] m_25[product_26] = m_26 m_acc[product_25] = m_25 m[account] = m_acc return m
def test_data_loop(self): dd1, dd2 = DummyDataset(), DummyDataset() dt1 = DummyTrainer(6, 2, deterministic_seed=50) dt2 = DummyTrainer(6, 2, deterministic_seed=50) self.assertTrue( torch.equal(dt1.model1.weight.data, dt2.model1.weight.data)) self.assertTrue( torch.equal(dt1.model2.weight.data, dt2.model2.weight.data)) self.assertTrue(torch.equal(dt1.const.data, dt2.const.data)) dt1 = DummyTrainer(6, 2, deterministic_seed=50) with suppress_stdout(): dt1.fit(dd1, dd2) dt2 = DummyTrainer(6, 2, deterministic_seed=50) with suppress_stdout(): dt2.fit(dd1, dd2) self.assertTrue( torch.equal(dt1.model1.weight.data, dt2.model1.weight.data)) self.assertTrue( torch.equal(dt1.model2.weight.data, dt2.model2.weight.data)) self.assertTrue(torch.equal(dt1.const.data, dt2.const.data))
def test_save_state(self): dt = DummyTrainer(1, 2, checkpoint_dir='tests/ex', save_frequency=1) dd1, dd2 = DummyDataset(), DummyDataset() with suppress_stdout(): dt.fit(dd1, dd2) dir_w_pths = os.listdir('tests/ex/epoch_1') self.assertIn('model1_epoch_1.pth', dir_w_pths) self.assertIn('model2_epoch_1.pth', dir_w_pths) self.assertNotIn('const_epoch_1.pth', dir_w_pths) self.assertNotIn('crit_epoch_1.pth', dir_w_pths) dir_w_pths = os.listdir('tests/ex/best_state') self.assertIn('model1_best_state.pth', dir_w_pths) self.assertIn('model2_best_state.pth', dir_w_pths) self.assertNotIn('const_best_state.pth', dir_w_pths) self.assertNotIn('crit_best_state.pth', dir_w_pths) shutil.rmtree('tests/ex')
def validate(strategy, cfg, model=None, split='val', clear_foot=False): cfg.DATASET.CACHE = False result_path = '{}/{}_{}.json'.format(cfg.MODEL.SAVE_DIR, cfg.MODEL.NAME, split) if split == 'val': with suppress_stdout(): coco = merge_coco_annotations(cfg.DATASET.ANNOT, split, clear_foot) if model is None: with strategy.scope(): model = tf.keras.models.load_model(osp.join( cfg.MODEL.SAVE_DIR, cfg.MODEL.NAME + '.h5'), compile=False) cfg.DATASET.OUTPUT_SHAPE = model.output_shape[1:] ds = load_tfds(cfg, split, det=cfg.VAL.DET, predict_kp=True, drop_remainder=cfg.VAL.DROP_REMAINDER) ds = strategy.experimental_distribute_dataset(ds) @tf.function def predict(imgs, flip=False): if flip: imgs = imgs[:, :, ::-1, :] return model(imgs, training=False) results = [] for count, batch in enumerate(ds): ids, imgs, _, Ms, scores = batch ids = np.concatenate(ids.values, axis=0) scores = np.concatenate(scores.values, axis=0) Ms = np.concatenate(Ms.values, axis=0) hms = strategy.run(predict, args=(imgs, )).values hms = np.array(np.concatenate(hms, axis=0), np.float32) if cfg.VAL.FLIP: flip_hms = strategy.run(predict, args=( imgs, True, )).values flip_hms = np.concatenate(flip_hms, axis=0) flip_hms = flip_hms[:, :, ::-1, :] tmp = flip_hms.copy() for i in range(len(cfg.DATASET.KP_FLIP)): flip_hms[:, :, :, i] = tmp[:, :, :, cfg.DATASET.KP_FLIP[i]] # shift to align features flip_hms[:, :, 1:, :] = flip_hms[:, :, 0:-1, :].copy() hms = (hms + flip_hms) / 2. preds = get_preds(hms, Ms, cfg.DATASET.INPUT_SHAPE, cfg.DATASET.OUTPUT_SHAPE) all_preds = np.zeros((preds.shape[0], 23, 3)) all_preds[:, :preds.shape[1], :] = preds preds = all_preds kp_scores = preds[:, :, -1].copy() # rescore rescored_score = np.zeros((len(kp_scores))) for i in range(len(kp_scores)): score_mask = kp_scores[i] > cfg.VAL.SCORE_THRESH if np.sum(score_mask) > 0: rescored_score[i] = np.mean( kp_scores[i][score_mask]) * scores[i] score_result = rescored_score for i in range(preds.shape[0]): results.append( dict(image_id=int(ids[i]), category_id=1, keypoints=preds[i].reshape(-1).tolist(), score=float(score_result[i]))) if cfg.TRAIN.DISP: print('completed preds batch', count + 1) with open(result_path, 'w') as f: json.dump(results, f) if split == 'val': with suppress_stdout(): result = coco.loadRes(result_path) cocoEval = COCOeval(coco, result, iouType='keypoints') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() mAP = cocoEval.stats[0] AP_50 = cocoEval.stats[1] AP_75 = cocoEval.stats[2] AP_small = cocoEval.stats[3] AP_medium = cocoEval.stats[4] AP_large = cocoEval.stats[5] return mAP, AP_50, AP_75, AP_small, AP_medium, AP_large # AP
def test_split(self, seed): if seed % 5 == 0: print("Test iteration {} of {}".format(seed + 1, self.number_tests)) self.train, self.test = self.test_splitting.get_split(seed, self.model.data) # Remove duped rows (possibly duplication has been provided) self.test = self.test.drop('cluster', axis=1).drop_duplicates() # Apply account test filter if not self.account_test_filter is None: self.test = self.test[self.test['account_banner'].isin(self.account_test_filter)] # Exclude products with large mape metric from test self.test = self.apply_product_filter(self.test, self.filter_data) # train in train_set and check the results in test set # split data into X, Y self.train_x = self.train.loc[:, self.train.columns != self.model.target] self.train_y = self.train.loc[:, self.train.columns == self.model.target] self.test_x = self.test.loc[:, self.test.columns != self.model.target] # test_x['baseline_units_original'] = test_x['baseline_units'] # test_x['baseline_units'] = test_x['baseline_units_2'] self.test_y = self.test.loc[:, self.test.columns == self.model.target] # transform to Series self.train_y = self.train_y.squeeze() self.test_y = self.test_y.squeeze() # Train model and validate performance with suppress_stdout(): self.model.model.fit(self.train_x, self.train_y) self.pred_train_y = self.model.model.predict(self.train_x) self.pred_test_y = self.model.model.predict(self.test_x) m = dict() metric_filter = ['r2_m', 'r2_h', 'r2_m_h', 'wape_m', 'wape_h', 'wape_m_h', 'mape_m', 'mape_h', 'mape_m_h', \ 'bias_m', 'bias_h', 'bias_m_h', 'sfa_m', 'sfa_h', 'sfa_m_h'] for account in self.test['account_banner'].unique(): m_acc = dict() metric_dict = self.get_metrics_by_account(metric_filter, account) m_acc['r2_m'] = metric_dict['r2_m'] m_acc['r2_h'] = metric_dict['r2_h'] m_acc['r2__m_h'] = metric_dict['r2_m_h'] m_acc['wape_m'] = metric_dict['wape_m'] m_acc['wape_h'] = metric_dict['wape_h'] m_acc['wape__m_h'] = metric_dict['wape_m_h'] m_acc['mape_m'] = metric_dict['mape_m'] m_acc['mape_h'] = metric_dict['mape_h'] m_acc['mape__m_h'] = metric_dict['mape_m_h'] m_acc['bias_m'] = metric_dict['bias_m'] m_acc['bias_h'] = metric_dict['bias_h'] m_acc['bias__m_h'] = metric_dict['bias_m_h'] m_acc['sfa_m'] = metric_dict['sfa_m'] m_acc['sfa_h'] = metric_dict['sfa_h'] m_acc['sfa__m_h'] = metric_dict['sfa_m_h'] m[account] = m_acc # Overall metrics m['seed'] = seed m['train_r2'] = round(r2_score(self.train_y, self.pred_train_y), 2) m['train'] = self.train_x.shape[0] m['test'] = self.test_x.shape[0] metric_dict = self.get_overall_metrics(metric_filter) m['r2_m'] = metric_dict['r2_m'] m['r2_h'] = metric_dict['r2_h'] m['r2__m_h'] = metric_dict['r2_m_h'] m['wape_m'] = metric_dict['wape_m'] m['wape_h'] = metric_dict['wape_h'] m['wape__m_h'] = metric_dict['wape_m_h'] m['mape_m'] = metric_dict['mape_m'] m['mape_h'] = metric_dict['mape_h'] m['mape__m_h'] = metric_dict['mape_m_h'] m['bias_m'] = metric_dict['bias_m'] m['bias_h'] = metric_dict['bias_h'] m['bias__m_h'] = metric_dict['bias_m_h'] m['sfa_m'] = metric_dict['sfa_m'] m['sfa_h'] = metric_dict['sfa_h'] m['sfa__m_h'] = metric_dict['sfa_m_h'] return m
if pretrained_seeker_weights: print('Loading pre-trained seeker') seeker = networks.seek.available_models[model_id](input_shape, num_classes) seeker.load_weights(pretrained_seeker_weights) if pretrained_hider_weights: print('Loading pre-trained hider') hider = networks.hide.available_models[model_id](input_shape) hider.load_weights(pretrained_hider_weights) print('Transfering weights') if model_id == 'hns_resnet': # This workaround is needed to transfer the seeker weights, # because the whole resnet model is represented as a layer with utils.suppress_stdout(): h, _ = utils.training.transfer_weights(hns_model, hider) s, _ = utils.training.transfer_weights( hns_model.get_layer('resnet50'), seeker) print('Transferred weights from {} hider and {} seeker layers.'. format(h, s)) else: utils.training.transfer_weights(hns_model, hider, seeker) del hider, seeker hns_trainer = HNSTrainer(model=hns_model, weight_dir=weight_dir, log_dir=log_dir, debug=debug, baseline=baseline)