def test_load_state(self):
dt = DummyTrainer(6, 2)
dd1, dd2 = DummyDataset(), DummyDataset()
with suppress_stdout():
dt.fit(dd1, dd2)
dt.test(dd1)
dt = DummyTrainer(6, 2, checkpoint_dir='tests/ex', save_frequency=3)
with suppress_stdout():
dt.fit(dd1, dd2)
dt.test(dd1, 'epoch_3')
shutil.rmtree('tests/ex')
def interactive(self, **kwargs):
with suppress_stdout():
args = self._default_args(**kwargs)
ans = interactive(args)
print(ans)
def speech2txt(buff, trigger):
resource_path = Path(os.getcwd()) / 'resources'
duration = 2
fs = 16000
nsample = duration * fs
nclass = 2
try:
trigger.set()
with utils.suppress_stdout():
log_speech.info("loading speech2txt model")
m = model.full_model(fs,
nsample,
nclass=nclass,
n_hop=128,
load_model=True,
resource_path=resource_path)
while trigger.wait():
log_speech.info("speech prediction is triggered")
print("choose right or left after the beep")
time.sleep(1)
audio = record_audio.record(duration, fs, resource_path)
pred = 'r' if predict.predict(m, audio) else 'l'
buff.put(pred, block=True)
log_speech.info('model predicted %s' % pred)
trigger.clear()
except TypeError as e:
log_speech.error('Failed to import utils, audio, or nn')
log_speech.error(str(e))
except KeyboardInterrupt as e:
log_speech.info("Forcebly closed by user: %s" % str(e))
def load_block(block_file):
with open(block_file) as infile:
with utils.suppress_stdout():
# data_block, numpy array of shape: [B x 3 x H x W]
data_block = hickle.load(infile)
# data_block, numpy array of shape: [B x H x W x 3]
data_block = np.transpose(data_block, (0, 2, 3, 1)) # to HWC
return data_block
def __mape_data_detailed_test_split(self, seed):
if seed % 5 == 0:
print("Test iteration {} of {}".format(seed + 1, self.number_tests))
self.train, self.test = self.test_splitting.get_split(seed, self.model.data)
# Remove duped rows (possibly duplication has been provided)
self.test = self.test.drop('cluster', axis=1).drop_duplicates()
# Apply account test filter
if not self.account_test_filter is None:
self.test = self.test[self.test['account_banner'].isin(self.account_test_filter)]
self.train_x = self.train.loc[:, self.train.columns != self.model.target]
self.train_y = self.train.loc[:, self.train.columns == self.model.target]
self.test_x = self.test.loc[:, self.test.columns != self.model.target]
# test_x['baseline_units_original'] = test_x['baseline_units']
# test_x['baseline_units'] = test_x['baseline_units_2']
self.test_y = self.test.loc[:, self.test.columns == self.model.target]
# transform to Series
self.train_y = self.train_y.squeeze()
self.test_y = self.test_y.squeeze()
# Train model and validate performance
with suppress_stdout():
self.model.model.fit(self.train_x, self.train_y)
self.pred_train_y = self.model.model.predict(self.train_x)
self.pred_test_y = self.model.model.predict(self.test_x)
m = dict()
for account in self.test['account_banner'].unique():
m_acc = dict()
for product_25 in self.test[self.test['account_banner']==account]['original_product_dimension_25'].unique():
m_25 = dict()
for product_26 in self.test[(self.test['account_banner']==account) & \
(self.test['original_product_dimension_25']==product_25)] \
['original_product_dimension_26'].unique():
m_26 = dict()
metric_dict = self.get_metrics_by_account_product25_product26(['mape_m', 'mape_h'],
account, product_25, product_26)
m_26['mape_m'] = metric_dict['mape_m']
m_26['mape_h'] = metric_dict['mape_h']
m_25[product_26] = m_26
m_acc[product_25] = m_25
m[account] = m_acc
return m
def test_data_loop(self):
dd1, dd2 = DummyDataset(), DummyDataset()
dt1 = DummyTrainer(6, 2, deterministic_seed=50)
dt2 = DummyTrainer(6, 2, deterministic_seed=50)
self.assertTrue(
torch.equal(dt1.model1.weight.data, dt2.model1.weight.data))
self.assertTrue(
torch.equal(dt1.model2.weight.data, dt2.model2.weight.data))
self.assertTrue(torch.equal(dt1.const.data, dt2.const.data))
dt1 = DummyTrainer(6, 2, deterministic_seed=50)
with suppress_stdout():
dt1.fit(dd1, dd2)
dt2 = DummyTrainer(6, 2, deterministic_seed=50)
with suppress_stdout():
dt2.fit(dd1, dd2)
self.assertTrue(
torch.equal(dt1.model1.weight.data, dt2.model1.weight.data))
self.assertTrue(
torch.equal(dt1.model2.weight.data, dt2.model2.weight.data))
self.assertTrue(torch.equal(dt1.const.data, dt2.const.data))
def test_save_state(self):
dt = DummyTrainer(1, 2, checkpoint_dir='tests/ex', save_frequency=1)
dd1, dd2 = DummyDataset(), DummyDataset()
with suppress_stdout():
dt.fit(dd1, dd2)
dir_w_pths = os.listdir('tests/ex/epoch_1')
self.assertIn('model1_epoch_1.pth', dir_w_pths)
self.assertIn('model2_epoch_1.pth', dir_w_pths)
self.assertNotIn('const_epoch_1.pth', dir_w_pths)
self.assertNotIn('crit_epoch_1.pth', dir_w_pths)
dir_w_pths = os.listdir('tests/ex/best_state')
self.assertIn('model1_best_state.pth', dir_w_pths)
self.assertIn('model2_best_state.pth', dir_w_pths)
self.assertNotIn('const_best_state.pth', dir_w_pths)
self.assertNotIn('crit_best_state.pth', dir_w_pths)
shutil.rmtree('tests/ex')
def validate(strategy, cfg, model=None, split='val', clear_foot=False):
cfg.DATASET.CACHE = False
result_path = '{}/{}_{}.json'.format(cfg.MODEL.SAVE_DIR, cfg.MODEL.NAME,
split)
if split == 'val':
with suppress_stdout():
coco = merge_coco_annotations(cfg.DATASET.ANNOT, split, clear_foot)
if model is None:
with strategy.scope():
model = tf.keras.models.load_model(osp.join(
cfg.MODEL.SAVE_DIR, cfg.MODEL.NAME + '.h5'),
compile=False)
cfg.DATASET.OUTPUT_SHAPE = model.output_shape[1:]
ds = load_tfds(cfg,
split,
det=cfg.VAL.DET,
predict_kp=True,
drop_remainder=cfg.VAL.DROP_REMAINDER)
ds = strategy.experimental_distribute_dataset(ds)
@tf.function
def predict(imgs, flip=False):
if flip:
imgs = imgs[:, :, ::-1, :]
return model(imgs, training=False)
results = []
for count, batch in enumerate(ds):
ids, imgs, _, Ms, scores = batch
ids = np.concatenate(ids.values, axis=0)
scores = np.concatenate(scores.values, axis=0)
Ms = np.concatenate(Ms.values, axis=0)
hms = strategy.run(predict, args=(imgs, )).values
hms = np.array(np.concatenate(hms, axis=0), np.float32)
if cfg.VAL.FLIP:
flip_hms = strategy.run(predict, args=(
imgs,
True,
)).values
flip_hms = np.concatenate(flip_hms, axis=0)
flip_hms = flip_hms[:, :, ::-1, :]
tmp = flip_hms.copy()
for i in range(len(cfg.DATASET.KP_FLIP)):
flip_hms[:, :, :, i] = tmp[:, :, :, cfg.DATASET.KP_FLIP[i]]
# shift to align features
flip_hms[:, :, 1:, :] = flip_hms[:, :, 0:-1, :].copy()
hms = (hms + flip_hms) / 2.
preds = get_preds(hms, Ms, cfg.DATASET.INPUT_SHAPE,
cfg.DATASET.OUTPUT_SHAPE)
all_preds = np.zeros((preds.shape[0], 23, 3))
all_preds[:, :preds.shape[1], :] = preds
preds = all_preds
kp_scores = preds[:, :, -1].copy()
# rescore
rescored_score = np.zeros((len(kp_scores)))
for i in range(len(kp_scores)):
score_mask = kp_scores[i] > cfg.VAL.SCORE_THRESH
if np.sum(score_mask) > 0:
rescored_score[i] = np.mean(
kp_scores[i][score_mask]) * scores[i]
score_result = rescored_score
for i in range(preds.shape[0]):
results.append(
dict(image_id=int(ids[i]),
category_id=1,
keypoints=preds[i].reshape(-1).tolist(),
score=float(score_result[i])))
if cfg.TRAIN.DISP:
print('completed preds batch', count + 1)
with open(result_path, 'w') as f:
json.dump(results, f)
if split == 'val':
with suppress_stdout():
result = coco.loadRes(result_path)
cocoEval = COCOeval(coco, result, iouType='keypoints')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
mAP = cocoEval.stats[0]
AP_50 = cocoEval.stats[1]
AP_75 = cocoEval.stats[2]
AP_small = cocoEval.stats[3]
AP_medium = cocoEval.stats[4]
AP_large = cocoEval.stats[5]
return mAP, AP_50, AP_75, AP_small, AP_medium, AP_large # AP
def test_split(self, seed):
if seed % 5 == 0:
print("Test iteration {} of {}".format(seed + 1, self.number_tests))
self.train, self.test = self.test_splitting.get_split(seed, self.model.data)
# Remove duped rows (possibly duplication has been provided)
self.test = self.test.drop('cluster', axis=1).drop_duplicates()
# Apply account test filter
if not self.account_test_filter is None:
self.test = self.test[self.test['account_banner'].isin(self.account_test_filter)]
# Exclude products with large mape metric from test
self.test = self.apply_product_filter(self.test, self.filter_data)
# train in train_set and check the results in test set
# split data into X, Y
self.train_x = self.train.loc[:, self.train.columns != self.model.target]
self.train_y = self.train.loc[:, self.train.columns == self.model.target]
self.test_x = self.test.loc[:, self.test.columns != self.model.target]
# test_x['baseline_units_original'] = test_x['baseline_units']
# test_x['baseline_units'] = test_x['baseline_units_2']
self.test_y = self.test.loc[:, self.test.columns == self.model.target]
# transform to Series
self.train_y = self.train_y.squeeze()
self.test_y = self.test_y.squeeze()
# Train model and validate performance
with suppress_stdout():
self.model.model.fit(self.train_x, self.train_y)
self.pred_train_y = self.model.model.predict(self.train_x)
self.pred_test_y = self.model.model.predict(self.test_x)
m = dict()
metric_filter = ['r2_m', 'r2_h', 'r2_m_h', 'wape_m', 'wape_h', 'wape_m_h', 'mape_m', 'mape_h', 'mape_m_h', \
'bias_m', 'bias_h', 'bias_m_h', 'sfa_m', 'sfa_h', 'sfa_m_h']
for account in self.test['account_banner'].unique():
m_acc = dict()
metric_dict = self.get_metrics_by_account(metric_filter, account)
m_acc['r2_m'] = metric_dict['r2_m']
m_acc['r2_h'] = metric_dict['r2_h']
m_acc['r2__m_h'] = metric_dict['r2_m_h']
m_acc['wape_m'] = metric_dict['wape_m']
m_acc['wape_h'] = metric_dict['wape_h']
m_acc['wape__m_h'] = metric_dict['wape_m_h']
m_acc['mape_m'] = metric_dict['mape_m']
m_acc['mape_h'] = metric_dict['mape_h']
m_acc['mape__m_h'] = metric_dict['mape_m_h']
m_acc['bias_m'] = metric_dict['bias_m']
m_acc['bias_h'] = metric_dict['bias_h']
m_acc['bias__m_h'] = metric_dict['bias_m_h']
m_acc['sfa_m'] = metric_dict['sfa_m']
m_acc['sfa_h'] = metric_dict['sfa_h']
m_acc['sfa__m_h'] = metric_dict['sfa_m_h']
m[account] = m_acc
# Overall metrics
m['seed'] = seed
m['train_r2'] = round(r2_score(self.train_y, self.pred_train_y), 2)
m['train'] = self.train_x.shape[0]
m['test'] = self.test_x.shape[0]
metric_dict = self.get_overall_metrics(metric_filter)
m['r2_m'] = metric_dict['r2_m']
m['r2_h'] = metric_dict['r2_h']
m['r2__m_h'] = metric_dict['r2_m_h']
m['wape_m'] = metric_dict['wape_m']
m['wape_h'] = metric_dict['wape_h']
m['wape__m_h'] = metric_dict['wape_m_h']
m['mape_m'] = metric_dict['mape_m']
m['mape_h'] = metric_dict['mape_h']
m['mape__m_h'] = metric_dict['mape_m_h']
m['bias_m'] = metric_dict['bias_m']
m['bias_h'] = metric_dict['bias_h']
m['bias__m_h'] = metric_dict['bias_m_h']
m['sfa_m'] = metric_dict['sfa_m']
m['sfa_h'] = metric_dict['sfa_h']
m['sfa__m_h'] = metric_dict['sfa_m_h']
return m
if pretrained_seeker_weights:
print('Loading pre-trained seeker')
seeker = networks.seek.available_models[model_id](input_shape,
num_classes)
seeker.load_weights(pretrained_seeker_weights)
if pretrained_hider_weights:
print('Loading pre-trained hider')
hider = networks.hide.available_models[model_id](input_shape)
hider.load_weights(pretrained_hider_weights)
print('Transfering weights')
if model_id == 'hns_resnet':
# This workaround is needed to transfer the seeker weights,
# because the whole resnet model is represented as a layer
with utils.suppress_stdout():
h, _ = utils.training.transfer_weights(hns_model, hider)
s, _ = utils.training.transfer_weights(
hns_model.get_layer('resnet50'), seeker)
print('Transferred weights from {} hider and {} seeker layers.'.
format(h, s))
else:
utils.training.transfer_weights(hns_model, hider, seeker)
del hider, seeker
hns_trainer = HNSTrainer(model=hns_model,
weight_dir=weight_dir,
log_dir=log_dir,
debug=debug,
baseline=baseline)
