def __init__(self, params):
self.params = params
self.word2vec = KeyedVectors.load_word2vec_format(params["word2vec"], binary=True)
self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
self.model = Model(params, self.device).to(self.device)
train_dataset = Loader(params, params['train_data'], self.word2vec, flag=True)
val_dataset = Loader(params, params['val_data'], self.word2vec)
test_dataset = Loader(params, params['test_data'], self.word2vec)
self.train_loader = DataLoader(dataset=train_dataset, batch_size=64, shuffle=True)
self.val_loader = DataLoader(dataset=val_dataset, batch_size=64, shuffle=False)
self.test_loader = DataLoader(dataset=test_dataset, batch_size=64, shuffle=False)
weight_p, bias_p = [], []
for name, p in self.model.named_parameters():
if 'bias' in name:
bias_p += [p]
else:
weight_p += [p]
self.optimizer = torch.optim.Adam([{'params': weight_p, 'weight_decay': 0},
{'params': bias_p, 'weight_decay': 0}
], lr=self.params['learning_rate'])
self.model_path = os.path.join(self.params['cache_dir'])
if not os.path.exists(self.model_path):
print('create path: ', self.model_path)
os.makedirs(self.model_path)
self.best_model = None
self.lr_epoch = 0
def infer_step(self, model: Model, inputs):
"""Run one inference step."""
predictions = model.infer_step(inputs)
outputs = self._post_process_infer_output(predictions)
return outputs
def eval_step(self, model: Model, inputs):
"""Run one evaluation step"""
return model.eval_step(inputs)
def train_step(self, model: Model, inputs):
"""Run one training step."""
return model.train_step(inputs)
}
}
def gen_classifier_loader(name, d):
def classifier_loader():
if name == 'googlenet/inceptionv1':
model = torch_models.__dict__[d['arch']](pretrained=False,
aux_logits=False,
transform_input=True)
else:
model = torch_models.__dict__[d['arch']](pretrained=False)
load_model_state_dict(model, name)
return model
return classifier_loader
for name, d in model_params.items():
registry.add_model(
Model(name=name,
arch=d['arch'],
transform=StandardTransform(d['img_resize_size'],
d['img_crop_size']),
normalization=StandardNormalization(d['mean'], d['std'],
d['input_space']),
classifier_loader=gen_classifier_loader(name, d),
eval_batch_size=d['eval_batch_size'],
adversarial_batch_size=d['adversarial_batch_size']
if 'adversarial_batch_size' in d else None))
class Trainer(object):
def __init__(self, params):
self.params = params
self.word2vec = KeyedVectors.load_word2vec_format(params["word2vec"], binary=True)
self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
self.model = Model(params, self.device).to(self.device)
train_dataset = Loader(params, params['train_data'], self.word2vec, flag=True)
val_dataset = Loader(params, params['val_data'], self.word2vec)
test_dataset = Loader(params, params['test_data'], self.word2vec)
self.train_loader = DataLoader(dataset=train_dataset, batch_size=64, shuffle=True)
self.val_loader = DataLoader(dataset=val_dataset, batch_size=64, shuffle=False)
self.test_loader = DataLoader(dataset=test_dataset, batch_size=64, shuffle=False)
weight_p, bias_p = [], []
for name, p in self.model.named_parameters():
if 'bias' in name:
bias_p += [p]
else:
weight_p += [p]
self.optimizer = torch.optim.Adam([{'params': weight_p, 'weight_decay': 0},
{'params': bias_p, 'weight_decay': 0}
], lr=self.params['learning_rate'])
self.model_path = os.path.join(self.params['cache_dir'])
if not os.path.exists(self.model_path):
print('create path: ', self.model_path)
os.makedirs(self.model_path)
self.best_model = None
self.lr_epoch = 0
# When iteration starts, queue and thread start to load data from files.
def train(self):
print('Trainnning begins......')
best_epoch_acc = 0
best_epoch_id = 0
print('=================================')
print('Model Params:')
print(self.params)
print('=================================')
self.evaluate(self.test_loader)
for i_epoch in range(self.params['max_epoches']):
self.model.train()
t_begin = time.time()
avg_batch_loss = self.train_one_epoch(i_epoch)
t_end = time.time()
print('Epoch %d ends. Average loss %.3f. %.3f seconds/epoch' % (i_epoch, avg_batch_loss, t_end - t_begin))
if i_epoch % self.params['evaluate_interval'] == 0 and i_epoch != 0:
print('=================================')
print('Overall evaluation')
# print('=================================')
# print('train set evaluation')
# train_acc = self.evaluate(self.train_loader)
print('=================================')
print('valid set evaluation')
valid_acc = self.evaluate(self.val_loader)
print('=================================')
print('test set evaluation')
test_acc = self.evaluate(self.test_loader)
print('=================================')
else:
print('=================================')
print('valid set evaluation')
valid_acc = self.evaluate(self.val_loader)
print('=================================')
if valid_acc > best_epoch_acc:
best_epoch_acc = valid_acc
best_epoch_id = i_epoch
print('Saving new best model...')
timestamp = time.strftime("%m%d%H%M%S", time.localtime())
self.best_model = self.model_path + timestamp + '.ckpt'
torch.save(self.model.state_dict(), self.best_model)
else:
if i_epoch - best_epoch_id >= self.params['early_stopping']:
print('Early stopped. Best loss %.3f at epoch %d' % (best_epoch_acc, best_epoch_id))
break
print('=================================')
print('Evaluating best model in file', self.best_model, '...')
if self.best_model is not None:
self.model.load_state_dict(torch.load(self.best_model))
self.evaluate(self.test_loader)
else:
print('ERROR: No checkpoint available!')
def train_one_epoch(self, i_epoch):
loss_sum = 0
t1 = time.time()
for i_batch, (frame_vecs, frame_n, ques_vecs, ques_n, labels, regs, idxs, windows, gt_windows) in enumerate(self.train_loader):
frame_vecs = frame_vecs.to(self.device)
frame_n = frame_n.to(self.device)
ques_vecs = ques_vecs.to(self.device)
ques_n = ques_n.to(self.device)
labels = labels.to(self.device)
regs = regs.to(self.device)
idxs = idxs.to(self.device)
# Forward pass
batch_loss, predict_score, predict_reg = self.model(frame_vecs, frame_n, ques_vecs, ques_n, labels, regs, idxs)
# Backward and optimize
self.optimizer.zero_grad()
batch_loss.backward()
torch.nn.utils.clip_grad_value_(self.model.parameters(), 1)
self.optimizer.step()
self.lr_epoch += 1
loss_sum += batch_loss.item()
if i_batch % self.params['display_batch_interval'] == 0 and i_batch != 0:
t2 = time.time()
print('Epoch %d, Batch %d, loss = %.4f, %.3f seconds/batch' % ( i_epoch, i_batch, loss_sum / i_batch ,
(t2 - t1) / self.params['display_batch_interval']))
t1 = t2
if self.lr_epoch > 0 and self.lr_epoch % 3000 == 0:
self.adjust_learning_rate()
avg_batch_loss = loss_sum / i_batch
return avg_batch_loss
def evaluate(self, data_loader):
# IoU_thresh = [0.5,0.7]
# top1,top5
all_correct_num_topn_IoU = np.zeros(shape=[2,2],dtype=np.float32)
all_retrievd = 0.0
self.model.eval()
for i_batch, (frame_vecs, frame_n, ques_vecs, ques_n, labels, regs, idxs, windows, gt_windows) in enumerate(data_loader):
frame_vecs = frame_vecs.to(self.device)
frame_n = frame_n.to(self.device)
ques_vecs = ques_vecs.to(self.device)
ques_n = ques_n.to(self.device)
labels = labels.to(self.device)
regs = regs.to(self.device)
idxs = idxs.to(self.device)
batch_size = len(frame_vecs)
# Forward pass
batch_loss, predict_score, predict_reg = self.model(frame_vecs, frame_n, ques_vecs, ques_n, labels, regs, idxs)
predict_score = predict_score.detach().cpu().numpy()
predict_reg = predict_reg.detach().cpu().numpy()
for i in range(batch_size):
predict_windows = predict_reg[i] + windows[i]
result = criteria.compute_IoU_recall(predict_score[i], predict_windows, gt_windows[i])
all_correct_num_topn_IoU += result
all_retrievd += batch_size
avg_correct_num_topn_IoU = all_correct_num_topn_IoU / all_retrievd
print('=================================')
print(avg_correct_num_topn_IoU)
print('=================================')
acc = avg_correct_num_topn_IoU[0,0]
return acc
def adjust_learning_rate(self, decay_rate=0.8):
for param_group in self.optimizer.param_groups:
param_group['lr'] = param_group['lr'] * decay_rate
import torchvision.models as torch_models
from registry import registry
from models.model_base import Model, StandardTransform, StandardNormalization
from mldb.utils import load_model_state_dict
def classifier_loader():
model = torch_models.resnet50()
load_model_state_dict(model, 'resnet50_augmix')
return model
registry.add_model(
Model(
name='resnet50_augmix',
arch='resnet50',
transform=StandardTransform(img_resize_size=256, img_crop_size=224),
normalization=StandardNormalization(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
classifier_loader=classifier_loader,
eval_batch_size=256,
))
wups_count2[i] / type_count[i]
for i in range(self.params['n_types'])
]
type_bleu1_acc = [
bleu1_count[i] / type_count[i]
for i in range(self.params['n_types'])
]
print('Wup@0 for each type:', type_wup_acc)
print('[email protected] for each type:', type_wup_acc2)
print('Bleu1 for each type:', type_bleu1_acc)
print('type count: ', type_count)
print('all test time: ', all_batch_time)
return bleu1_acc
def _test(self, sess):
print('Validation set:')
valid_acc = self._evaluate(sess, self.model, self.valid_batcher)
print('Test set:')
test_acc = self._evaluate(sess, self.model, self.test_batcher)
return 0.0, valid_acc, test_acc
if __name__ == '__main__':
config_file = '../configs/config_base.json'
with open(config_file, 'r') as fr:
config = json.load(fr)
# print(config)
model = Model(config)
trainer = Trainer(config, model)
trainer.train()
'adversarial_batch_size': 1},
'BiT-M-R101x3-ILSVRC2012': { 'arch': 'BiT-M-R101x3',
'eval_batch_size': 32,
'adversarial_batch_size': 1},
'BiT-M-R152x4-ILSVRC2012': { 'arch': 'BiT-M-R152x4',
'eval_batch_size': 8},
}
def gen_classifier_loader(name, d):
def classifier_loader():
model = KNOWN_MODELS[d['arch']](head_size=1000)
load_model_state_dict(model, name)
return model
return classifier_loader
for name, d in model_params.items():
registry.add_model(
Model(
name = name,
arch = d['arch'],
transform = transforms.Compose([transforms.Resize((480, 480)),
transforms.ToTensor()]),
normalization = StandardNormalization([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),
classifier_loader = gen_classifier_loader(name, d),
eval_batch_size = d['eval_batch_size'],
adversarial_batch_size = d['adversarial_batch_size'] if 'adversarial_batch_size' in d else None,
)
)
model = EfficientNet.from_name(d['arch'])
load_model_state_dict(model, name)
return model
return classifier_loader
for name, d in model_params.items():
registry.add_model(
Model(name=name,
arch=d['arch'],
transform=transforms.Compose([
transforms.Resize(d['img_size'] + CROP_PADDING,
interpolation=PIL.Image.BICUBIC),
transforms.CenterCrop(d['img_size']),
transforms.ToTensor()
]),
normalization=StandardNormalization(d['mean'], d['std']),
classifier_loader=gen_classifier_loader(name, d),
eval_batch_size=d['eval_batch_size'],
adversarial_batch_size=d['adversarial_batch_size']
if 'adversarial_batch_size' in d else None))
def noisystudent_loader():
model = timm.create_model('tf_efficientnet_l2_ns', pretrained=False)
load_model_state_dict(model, 'efficientnet-l2-noisystudent')
return model
registry.add_model(
'lstm_dim': 384,
'second_lstm_dim': 384,
'attention_dim': 256,
'regularization_beta': 1e-7,
'dropout_prob': 0.6
}
data_params = {
'batch_size': 100,
'n_classes': 5032,
'n_types': 5,
'input_video_dim': 202,
'max_n_frames': 240,
'max_n_q_words': 47,
'input_ques_dim': 300,
'ref_dim': 300,
'word2vec': '../../data/word2vec/word2vec.bin',
'database': '../data/actNet200-V1-3.pkl',
'question': '../data/question.json',
'answer': '../data/answer.json',
'real_train_proposals': '../data/train.json',
'real_val_proposals': '../data/val_1.json',
'real_test_proposals': '../data/val_2.json',
'feature_path': '../../data/tsn_score/'
}
if __name__ == '__main__':
model = Model(data_params, model_params)
trainer = Trainer(train_params, model_params, data_params, model)
trainer.train()
def gen_classifier_loader(name, d):
def classifier_loader():
model = torch_models.__dict__[d['arch']]()
load_model_state_dict(model, name)
model = Smooth(model, d['noise_sigma'], d['n'], d['alpha'], d['mean'],
d['std'])
return model
return classifier_loader
def classify(images, model, class_sublist, adversarial_attack):
if adversarial_attack:
images = pgd_style_attack(adversarial_attack, images, model)
return model.predict_batch(images, class_sublist=class_sublist)
for name, d in model_params.items():
registry.add_model(
Model(
name=name,
arch=d['arch'],
transform=StandardTransform(d['img_resize_size'],
d['img_crop_size']),
classifier_loader=gen_classifier_loader(name, d),
eval_batch_size=d['eval_batch_size'],
adversarial_batch_size=d['adversarial_batch_size']
if 'adversarial_batch_size' in d else None,
classify=classify,
))
def eval_step(self, model: Model, inputs):
"""Run one evaluation step"""
outputs = model.eval_step(inputs)
outputs = {k: v.tolist()[0] for k, v in outputs.items()}
return outputs
def train_step(self, model: Model, inputs):
"""Run one training step."""
outputs = model.train_step(inputs)
outputs = {k: v.tolist()[0] for k, v in outputs.items()}
return outputs
model = pretrainedmodels.__dict__[d['arch']](num_classes=1000,
pretrained=None)
load_model_state_dict(model, name)
return model
return classifier_loader
for name, d in model_params.items():
registry.add_model(
Model(
name=name,
arch=d['arch'],
transform=StandardTransform(d['img_resize_size'],
d['img_crop_size']),
normalization=StandardNormalization(d['mean'], d['std'],
d['input_space']),
classifier_loader=gen_classifier_loader(name, d),
eval_batch_size=d['eval_batch_size'],
adversarial_batch_size=d['adversarial_batch_size']
if 'adversarial_batch_size' in d else None,
))
for name, d in squeezenets.items():
registry.add_model(
Model(name=name,
arch=d['arch'],
transform=StandardTransform(d['img_resize_size'],
d['img_crop_size']),
classifier_loader=gen_classifier_loader(name, d),
eval_batch_size=d['eval_batch_size'],
normalization=StandardNormalization(d['mean'], d['std'],
