def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
# prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = ""
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/thumos14_checkpoint_best_cas_epoch125_iou0.5__0.2928.pth"
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet12_checkpoint_best_cas_epoch30_map_0.2394.pth"
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet13_checkpoint_best_cas_epoch35_map_0.2348.pth"
weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/thumos14/000_thumos_29.13_save_model_Frame_wise_accuracy/checkpoint_best_cas.pth"
epoch = 801
from utils.utils import load_weights
model = load_weights(model, weight_file)
evaluate_save_for_post_process(cfg, val_loader, model, epoch)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
cfg.BASIC.ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
model.cuda()
if not os.path.exists(args.res_dir):
os.makedirs(args.res_dir)
model = load_weights(model, args.weight_file)
evaluate_vis_cas(cfg, val_loader, model, args.res_dir, args.is_minmax_norm)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
# prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model_cas = LocNet(cfg)
# model.apply(weight_init)
model_cam = LocNet(cfg)
model_cas.cuda()
model_cam.cuda()
# weight_file = ""
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/thumos14_checkpoint_best_cas_epoch125_iou0.5__0.2928.pth"
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet12_checkpoint_best_cas_epoch30_map_0.2394.pth"
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/anet13/anet13_same_as_anet12_seed1_epoch45_TOPK_K_R_0.6_LR_DECAY26_save_every_model/checkpoint_best_cas_epoch35.pth"
weight_file_cas = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/thumos14/thumos_ablation_only_cas_only_cam_separate_weight_save_model/checkpoint_best_cas_epoch75_0.2055.pth"
weight_file_cam = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/thumos14/thumos_ablation_only_cas_only_cam_separate_weight_save_model/checkpoint_best_cam_epoch89_0.176.pth"
from utils.utils import load_weights
model_cas = load_weights(model_cas, weight_file_cas)
model_cam = load_weights(model_cam, weight_file_cam)
epoch = 911
output_json_file_cas, test_acc_cas = evaluate_fuse_sequence(
cfg, val_loader, model_cas, model_cam, epoch)
# output_json_file_cas, test_acc_cas = evaluate(cfg, val_loader, model, epoch)
if cfg.BASIC.VERBOSE:
print('test_acc, cas %f' % (test_acc_cas))
mAP, average_mAP = evaluate_mAP(
cfg, output_json_file_cas,
os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE),
cfg.BASIC.VERBOSE)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/thumos14_checkpoint_best_cas_epoch125_iou0.5__0.2928.pth'
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet12_checkpoint_best_cas_epoch30_map_0.2545.pth'
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet13_checkpoint_best_cas_epoch35_map_0.2348.pth'
# weight_file = ''
weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/thumos14/thumos_ablation_inv_0_save_model/checkpoint_best_cas_inv0_epoch69_0.2636.pth'
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/thumos14/thumos_ablation_only_cas_save_model/checkpoint_best_cas_only_cas_epoch134_0.1957.pth'
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/thumos14/thumos_ablation_individual_attention_2048k1_2048k1_2048k1_only_cam_svae_model/checkpoint_best_cas_only_cam_epoch96_0.1714.pth'
res_dir = os.path.join(cfg.BASIC.CKPT_DIR, cfg.TEST.RESULT_DIR,
'vis/cas_gt_idx_minmax_norm_std')
if not os.path.exists(res_dir):
os.makedirs(res_dir)
from utils.utils import load_weights
model = load_weights(model, weight_file)
epoch = 600
output_json_file_cas, output_json_file_cam, test_acc_cas, test_acc_cam = evaluate(
cfg, val_loader, model, epoch)
evaluate_mAP(cfg, output_json_file_cas,
os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE),
cfg.BASIC.VERBOSE)
evaluate_mAP(cfg, output_json_file_cam,
os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE),
cfg.BASIC.VERBOSE)
is_minmax_norm = True
evaluate_vis_cas_minmax_norm_std(cfg, val_loader, model, res_dir,
is_minmax_norm)
def main(config):
print(config)
# initialize model
model = get_model(config)
# load weights from checkpoint
state_dict = load_weights(config['resume_ckpt'])
model.load_state_dict(state_dict)
model.cuda()
model.eval()
train_dataloader, test_dataloader, val_dataloader = setup_dataloaders(
config)
eval_encoding(config, model, train_dataloader, test_dataloader,
val_dataloader)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = ""
weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/thumos14_checkpoint_best_cas_epoch125_iou0.5__0.2928.pth"
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet12_checkpoint_best_cas_epoch30_map_0.2394.pth"
# weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet13_checkpoint_best_cas_epoch35_map_0.2348.pth"
epoch = 801
from utils.utils import load_weights
model = load_weights(model, weight_file)
# actions_json_file = evaluate(cfg, val_loader, model, epoch)
#
# evaluate_mAP(cfg, actions_json_file, os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE))
# output_json_file_cas, output_json_file_cam, test_acc_cas, test_acc_cam = evaluate(cfg, val_loader, model, epoch)
output_json_file_cas, test_acc_cas = evaluate(cfg, val_loader, model,
epoch)
if cfg.BASIC.VERBOSE:
print('test_acc, cas %f' % (test_acc_cas))
mAP, average_mAP = evaluate_mAP(
cfg, output_json_file_cas,
os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE),
cfg.BASIC.VERBOSE)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
# prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/anet12/anet12_0.2350_cam_inv_1_seed7_epoch36_TOPK_K_R_0.25__save_model_LR_decay/anet12_checkpoint_best_cas_0.2394.pth'
weight_file = "/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/anet13/anet13_same_as_anet12_seed1_epoch45_TOPK_K_R_0.6_LR_DECAY26_save_every_model/anet13_checkpoint_best_cas_epoch28_0.2178.pth"
epoch = 603
from utils.utils import load_weights
model = load_weights(model, weight_file)
# actions_json_file = evaluate(cfg, val_loader, model, epoch)
#
# evaluate_mAP(cfg, actions_json_file, os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE))
# output_json_file_cas, output_json_file_cam, test_acc_cas, test_acc_cam = evaluate(cfg, val_loader, model, epoch)
output_json_file_cas, test_acc_cas = evaluate(cfg, val_loader, model,
epoch)
if cfg.BASIC.VERBOSE:
print('test_acc, cas %f' % (test_acc_cas))
mAP, average_mAP = evaluate_mAP(
cfg, output_json_file_cas,
os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE),
cfg.BASIC.VERBOSE)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset, batch_size=cfg.TEST.BATCH_SIZE, shuffle=False,
num_workers=cfg.BASIC.WORKERS, pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/thumos14_checkpoint_best_cas_epoch125_iou0.5__0.2928.pth'
# weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet12_checkpoint_best_cas_epoch30_map_0.2545.pth'
weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/0_NeurIPS2020_code_ok/results_and_model/anet13_checkpoint_best_cas_epoch35_map_0.2348.pth'
res_dir = os.path.join(cfg.BASIC.CKPT_DIR, cfg.TEST.RESULT_DIR,'vis/cas_weight_gt_idx_iou_0.85_mul_instance')
if not os.path.exists(res_dir):
os.makedirs(res_dir)
from utils.utils import load_weights
model = load_weights(model, weight_file)
# epoch = 600
# actions_json_file, _ = evaluate(cfg, val_loader, model, epoch)
# evaluate_mAP(cfg, actions_json_file, os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE), cfg.BASIC.VERBOSE)
is_minmax_norm = True
evaluate_vis_cas_select_specific(cfg, val_loader, model, res_dir, is_minmax_norm)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
weight_file = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/anet12/anet12_no_inv_base_LR_0.0002_BATCH_128_save_model/checkpoint_best_cas_0.2243.pth'
epoch = 600
res_dir = '/disk3/zt/code/4_a/1_ECM_no_inv_drop/output/anet12/anet12_no_inv_base_LR_0.0002_BATCH_128_save_model/vis/cas_minmax_norm'
if not os.path.exists(res_dir):
os.makedirs(res_dir)
from utils.utils import load_weights
model = load_weights(model, weight_file)
is_minmax_norm = True
# actions_json_file = evaluate(cfg, val_loader, model, epoch)
#
# evaluate_mAP(cfg, actions_json_file, os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE))
# output_json_file_cas, output_json_file_cam, test_acc_cas, test_acc_cam = evaluate(cfg, val_loader, model, epoch)
evaluate_vis_cas(cfg, val_loader, model, epoch, res_dir, is_minmax_norm)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = ''
weight_file = '/disk/yangle/Short-Actions/ECM/output/thumos14/ECM_baseline/checkpoint_best_150.pth'
res_dir = os.path.join(cfg.BASIC.CKPT_DIR, cfg.TEST.RESULT_DIR,
'vis/ECM_thumos_score')
if not os.path.exists(res_dir):
os.makedirs(res_dir)
from utils.utils import load_weights
model = load_weights(model, weight_file)
epoch = 600
# output_json_file_cas, test_acc_cas = evaluate(cfg, val_loader, model, epoch)
output_json_file_cas = '/disk/yangle/Short-Actions/ECM/output/thumos14/ECM_baseline/vis/ecm.json'
evaluate_mAP(cfg, output_json_file_cas,
os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE),
cfg.BASIC.VERBOSE)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
# prepare_env(cfg)
# dataloader
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset, batch_size=cfg.TEST.BATCH_SIZE, shuffle=False,
num_workers=cfg.BASIC.WORKERS, pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = ""
weight_file = "/disk/yangle/Short-Actions/ECM/output/thumos14/ECM_baseline/checkpoint_best_150.pth"
epoch = 601
from utils.utils import load_weights
model = load_weights(model, weight_file)
# actions_json_file = evaluate(cfg, val_loader, model, epoch)
#
# evaluate_mAP(cfg, actions_json_file, os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE))
# output_json_file_cas, output_json_file_cam, test_acc_cas, test_acc_cam = evaluate(cfg, val_loader, model, epoch)
output_json_file_cas, test_acc_cas = evaluate(cfg, val_loader, model, epoch)
if cfg.BASIC.VERBOSE:
print('test_acc, cas %f' % (test_acc_cas))
mAP, average_mAP = evaluate_mAP(cfg, output_json_file_cas, os.path.join(cfg.BASIC.CKPT_DIR, cfg.DATASET.GT_FILE), cfg.BASIC.VERBOSE)
model.add(LeakyReLU(alpha=0.1)) model.add(Convolution2D(1024, 3, 3, border_mode='same')) model.add(LeakyReLU(alpha=0.1)) model.add(Convolution2D(1024, 3, 3, border_mode='same')) model.add(LeakyReLU(alpha=0.1)) model.add(Flatten()) model.add(Dense(256)) model.add(Dense(4096)) model.add(LeakyReLU(alpha=0.1)) model.add(Dense(1470)) print(model.summary()) # https://pan.baidu.com/s/1o9twnPo load_weights(model, './yolo-tiny.weights') imagePath = './test_images/test1.jpg' image = plt.imread(imagePath) image_crop = image[300:650, 500:, :] resized = cv2.resize(image_crop, (448, 448)) batch = np.transpose(resized, (2, 0, 1)) batch = 2 * (batch / 255.) - 1 batch = np.expand_dims(batch, axis=0) out = model.predict(batch) boxes = yolo_net_out_to_car_boxes(out[0], threshold=0.17) f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6)) ax1.imshow(image)
def detect(image_path, model_path, yolo_weights=None):
"""
Introduction
------------
加载模型，进行预测
Parameters
----------
model_path: 模型路径
image_path: 图片路径
"""
image = Image.open(image_path)
resize_image = letterbox_image(image,
(416, 416)) #resize 和padding图像 与训练的时候保持一致
image_data = np.array(resize_image, dtype=np.float32)
image_data /= 255.
image_data = np.expand_dims(image_data, axis=0) #扩展为[1 416 416 3]用于输入网络
input_image_shape = tf.placeholder(dtype=tf.int32, shape=(2, ))
input_image = tf.placeholder(shape=[None, 416, 416, 3], dtype=tf.float32)
predictor = yolo_predictor(config.obj_threshold, config.nms_threshold,
config.classes_path, config.anchors_path)
boxes, scores, classes = predictor.predict(input_image, input_image_shape)
with tf.Session() as sess:
if yolo_weights is not None:
with tf.variable_scope('predict'):
boxes, scores, classes = predictor.predict(
input_image, input_image_shape)
load_op = load_weights(tf.global_variables(scope='predict'),
weights_file=yolo_weights)
sess.run(load_op)
else:
saver = tf.train.Saver()
saver.restore(sess, model_path)
#运行检测
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
input_image: image_data,
input_image_shape: [image.size[1], image.size[0]]
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(
out_classes))): #使用了reversed但是 枚举的索引与元素的对应关系没变，reversed意义何在？
predicted_class = predictor.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image) #使用PIL里面的函数
label_size = draw.textsize(label, font)
top, left, bottom, right = box #这里解析 box的形式是 y1 x1 y2 x2
top = max(
0,
np.floor(top + 0.5).astype(
'int32')) #因为是预测给出的框的大小，有可能越界，判断一下边界还是必要的，加0.5是为了四舍五入，向上取整
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([
left, top - label_size[1]
]) #标签的文本一般放在框的上面，如果框的上面的高度大于label_size[1]，则可以在上面画框，否则在下面画框
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=predictor.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=predictor.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
image.show()
image.save('./test/result1.jpg')
] #(# of training samples, number_of_frames, appearance_representation_size)
input_appearance_list_test = [
] #(# of testing samples, number_of_frames, appearance_representation_size)
input_flow_list_train = [
] #(# of training samples, number_of_frames, width, height, channels)
input_flow_list_test = [
] #(# of testing samples, number_of_frames, width, height, channels)
video_labels = [
] #(# of samples, number_of_classes) #this array must be categorical
#you can create a model from scratch or load a trained model
if load_mode == 0:
bubble_model = bm.create_bubble_network(batch_size, number_of_frames,
appearance_representation_size,
new_classes) #creating a model
else:
bubble_model = utils.load_weights(path_to_load_model)
#bubble_model.compile(optimizer=Adam(lr=0.0005, beta_1=0.9, beta_2=0.999), loss='categorical_crossentropy') #use this line if u need to compile the model
#if you need to fit the model, use this. The dimensions shown below are related to our already trained model. You can train from scratch with different values
bubble_model = bm.bubble_network_fit(bubble_model, input_flow_list_train,
input_appearance_list_train,
number_of_frames,
output_label_list_train,
number_of_epochs, batch_size,
number_of_classes)
#to predict samples -> parameters are (samples, 16, 225), (# of samples, 16, 112, 112, 3), batch_size
predictions = bubble_model.predict(
[input_appearance_list_test, input_flow_list_test], batch_size)
def train(args: Arguments) -> None:
"""Trains an AlignmentModel to align sets of sentences."""
if args.task == "classify":
from classify.compute import AlignmentTrainer
from classify.data import load_data
from classify.metric import load_loss_and_metrics
if args.word_to_word:
from classify.models.ot_atten import AlignmentModel
else:
from classify.models.ot_atten_sent import AlignmentModel
elif args.task == "similarity":
from similarity.compute import AlignmentTrainer
from similarity.data import load_data
from similarity.metric import load_loss_and_metrics
from similarity.models import AlignmentModel
# Determine device
device = (torch.device(args.gpu)
if torch.cuda.is_available() else torch.device("cpu"))
# device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
print("Loading data")
text_field, train_sampler, dev_sampler, test_sampler = load_data(
args, device)
print("Building model")
model = AlignmentModel(
args=args,
text_field=text_field,
domain=args.dataset,
device=device,
)
saved_step = 0
if args.checkpoint_path is not None:
print(f"Loading checkpoint from: {args.checkpoint_path}")
saved_step = 1 + int(
args.checkpoint_path.split("_")[-1].replace(".pt", ""))
print(f"trainig from step {saved_step}")
load_weights(model, args.checkpoint_path)
print(model)
print(f"Number of parameters = {model.num_parameters(trainable=True):,}")
print(f"Moving model to device: {device}")
model.to(device)
print("Defining loss and metrics")
(
loss_fn,
metric_fn,
extra_training_metrics,
extra_validation_metrics,
) = load_loss_and_metrics(args)
print("Creating optimizer and scheduler")
if args.bert:
# Prepare optimizer and schedule (linear warmup and decay)
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay":
args.weight_decay,
},
{
"params": [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0,
},
]
# optimizer_grouped_parameters = get_params(model)
optimizer = AdamW(optimizer_grouped_parameters, lr=args.lr, eps=1e-8)
# num_batch_per_epoch = min(train_data.num_batches, args.max_batches_per_epoch)
num_batch_per_epoch = len(train_sampler)
t_total = int(num_batch_per_epoch // args.gradient_accumulation_steps *
args.epochs)
# scheduler = WarmupLinearSchedule(optimizer, warmup_steps=int(t_total*0.06), t_total=t_total)
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=int(t_total * args.warmup_ratio),
num_training_steps=t_total,
)
else:
optimizer = Adam(model.trainable_params,
lr=args.lr,
weight_decay=args.weight_decay)
scheduler = make_schedular(args,
optimizer,
model.output_size,
last_epoch=saved_step - 1)
print("Building Trainer")
trainer = AlignmentTrainer(
args=args,
train_sampler=train_sampler,
dev_sampler=dev_sampler,
test_sampler=test_sampler,
model=model,
loss_fn=loss_fn,
metric_fn=metric_fn,
optimizer=optimizer,
scheduler=scheduler,
epochs=args.epochs,
extra_training_metrics=extra_training_metrics,
extra_validation_metrics=extra_validation_metrics,
log_dir=args.log_dir,
log_frequency=args.log_frequency,
gradient_accumulation_steps=args.gradient_accumulation_steps,
sparsity_thresholds=args.sparsity_thresholds,
saved_step=saved_step,
)
if args.epochs > 0:
print("Training")
while not trainer.step():
pass
if args.preds_dir is not None or args.viz_dir is not None:
print("Predicting")
sentences, preds, targets = trainer.predict(
num_predict=args.num_predict)
# Extract targets
targets = [target["targets"] for target in targets]
targets = [t.item() for target in targets for t in target]
# Convert indices back to tokens
sentences = [(
[text_field.deprocess(sentence) for sentence in doc_1],
[text_field.deprocess(sentence) for sentence in doc_2],
) for doc_1, doc_2 in sentences]
# Save predictions
if args.preds_dir is not None:
makedirs(args.preds_dir)
preds_path = os.path.join(args.preds_dir, "preds.pkl")
with open(preds_path, "wb") as f:
sentences, preds, targets = sentences, preds, targets
pickle.dump((sentences, preds, targets), f)
elif args.epochs == 0:
print("Evaluating")
trainer.eval_step()
def main(config):
save_path = config['save_path']
epochs = config['epochs']
os.environ['TORCH_HOME'] = config['torch_home']
distributed = config['use_DDP']
start_ep = 0
start_cnt = 0
# initialize model
print("Initializing model...")
if distributed:
initialize_distributed(config)
rank = config['rank']
# map string name to class constructor
model = get_model(config)
model.apply(init_weights)
if config['resume_ckpt'] is not None:
# load weights from checkpoint
state_dict = load_weights(config['resume_ckpt'])
model.load_state_dict(state_dict)
print("Moving model to GPU")
model.cuda(torch.cuda.current_device())
print("Setting up losses")
if config['use_vgg']:
criterionVGG = Vgg19PerceptualLoss(config['reduced_w'])
criterionVGG.cuda()
validationLoss = criterionVGG
if config['use_gan']:
use_sigmoid = config['no_lsgan']
disc_input_channels = 3
discriminator = MultiscaleDiscriminator(disc_input_channels,
config['ndf'],
config['n_layers_D'],
'instance', use_sigmoid,
config['num_D'], False, False)
discriminator.apply(init_weights)
if config['resume_ckpt_D'] is not None:
# load weights from checkpoint
print("Resuming discriminator from %s" % (config['resume_ckpt_D']))
state_dict = load_weights(config['resume_ckpt_D'])
discriminator.load_state_dict(state_dict)
discriminator.cuda(torch.cuda.current_device())
criterionGAN = GANLoss(use_lsgan=not config['no_lsgan'])
criterionGAN.cuda()
criterionFeat = nn.L1Loss().cuda()
if config['use_l2']:
criterionMSE = nn.MSELoss()
criterionMSE.cuda()
validationLoss = criterionMSE
# initialize dataloader
print("Setting up dataloaders...")
train_dataloader, val_dataloader, train_sampler = setup_dataloaders(config)
print("Done!")
# run the training loop
print("Initializing optimizers...")
optimizer_G = optim.Adam(model.parameters(),
lr=config['learning_rate'],
weight_decay=config['weight_decay'])
if config['resume_ckpt_opt_G'] is not None:
optimizer_G_state_dict = torch.load(
config['resume_ckpt_opt_G'],
map_location=lambda storage, loc: storage)
optimizer_G.load_state_dict(optimizer_G_state_dict)
if config['use_gan']:
optimizer_D = optim.Adam(discriminator.parameters(),
lr=config['learning_rate'])
if config['resume_ckpt_opt_D'] is not None:
optimizer_D_state_dict = torch.load(
config['resume_ckpt_opt_D'],
map_location=lambda storage, loc: storage)
optimizer_D.load_state_dict(optimizer_D_state_dict)
print("Done!")
if distributed:
print("Moving model to DDP...")
model = DDP(model)
if config['use_gan']:
discriminator = DDP(discriminator, delay_allreduce=True)
print("Done!")
tb_logger = None
if rank == 0:
tb_logdir = os.path.join(save_path, 'tbdir')
if not os.path.exists(tb_logdir):
os.makedirs(tb_logdir)
tb_logger = SummaryWriter(tb_logdir)
# run training
if not os.path.exists(save_path):
os.makedirs(save_path)
log_name = os.path.join(save_path, 'loss_log.txt')
opt_name = os.path.join(save_path, 'opt.yaml')
print(config)
save_options(opt_name, config)
log_handle = open(log_name, 'a')
print("Starting training")
cnt = start_cnt
assert (config['use_warped'] or config['use_temporal'])
for ep in range(start_ep, epochs):
if train_sampler is not None:
train_sampler.set_epoch(ep)
for curr_batch in train_dataloader:
optimizer_G.zero_grad()
input_a = curr_batch['input_a'].cuda()
target = curr_batch['target'].cuda()
if config['use_warped'] and config['use_temporal']:
input_a = torch.cat((input_a, input_a), 0)
input_b = torch.cat((curr_batch['input_b'].cuda(),
curr_batch['input_temporal'].cuda()), 0)
target = torch.cat((target, target), 0)
elif config['use_temporal']:
input_b = curr_batch['input_temporal'].cuda()
elif config['use_warped']:
input_b = curr_batch['input_b'].cuda()
output_dict = model(input_a, input_b)
output_recon = output_dict['reconstruction']
loss_vgg = loss_G_GAN = loss_G_feat = loss_l2 = 0
if config['use_vgg']:
loss_vgg = criterionVGG(output_recon,
target) * config['vgg_lambda']
if config['use_gan']:
predicted_landmarks = output_dict['input_a_gauss_maps']
# output_dict['reconstruction'] can be considered normalized
loss_G_GAN, loss_D_real, loss_D_fake = apply_GAN_criterion(
output_recon, target, predicted_landmarks.detach(),
discriminator, criterionGAN)
loss_D = (loss_D_fake + loss_D_real) * 0.5
if config['use_l2']:
loss_l2 = criterionMSE(output_recon,
target) * config['l2_lambda']
loss_G = loss_G_GAN + loss_G_feat + loss_vgg + loss_l2
loss_G.backward()
# grad_norm clipping
if not config['no_grad_clip']:
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer_G.step()
if config['use_gan']:
optimizer_D.zero_grad()
loss_D.backward()
# grad_norm clipping
if not config['no_grad_clip']:
torch.nn.utils.clip_grad_norm_(discriminator.parameters(),
1.0)
optimizer_D.step()
if distributed:
if config['use_vgg']:
loss_vgg = reduce_tensor(loss_vgg, config['world_size'])
if rank == 0:
if cnt % 10 == 0:
run_visualization(output_dict, output_recon, target,
input_a, input_b, save_path, tb_logger,
cnt)
print_dict = {"learning_rate": get_learning_rate(optimizer_G)}
if config['use_vgg']:
tb_logger.add_scalar('vgg.loss', loss_vgg, cnt)
print_dict['Loss_VGG'] = loss_vgg.data
if config['use_gan']:
tb_logger.add_scalar('gan.loss', loss_G_GAN, cnt)
tb_logger.add_scalar('d_real.loss', loss_D_real, cnt)
tb_logger.add_scalar('d_fake.loss', loss_D_fake, cnt)
print_dict['Loss_G_GAN'] = loss_G_GAN
print_dict['Loss_real'] = loss_D_real.data
print_dict['Loss_fake'] = loss_D_fake.data
if config['use_l2']:
tb_logger.add_scalar('l2.loss', loss_l2, cnt)
print_dict['Loss_L2'] = loss_l2.data
log_iter(ep,
cnt % len(train_dataloader),
len(train_dataloader),
print_dict,
log_handle=log_handle)
if loss_G != loss_G:
print("NaN!!")
exit(-2)
cnt = cnt + 1
# end of train iter loop
if cnt % config['val_freq'] == 0 and config['val_freq'] > 0:
val_loss = run_val(
model, validationLoss, val_dataloader,
os.path.join(save_path, 'val_%d_renders' % (ep)))
if distributed:
val_loss = reduce_tensor(val_loss, config['world_size'])
if rank == 0:
tb_logger.add_scalar('validation.loss', val_loss, cnt)
log_iter(ep,
cnt % len(train_dataloader),
len(train_dataloader), {"Loss_VGG": val_loss},
header="Validation loss: ",
log_handle=log_handle)
if rank == 0:
if (ep % config['save_freq'] == 0):
fname = 'checkpoint_%d.ckpt' % (ep)
fname = os.path.join(save_path, fname)
print("Saving model...")
save_weights(model, fname, distributed)
optimizer_g_fname = os.path.join(
save_path, 'latest_optimizer_g_state.ckpt')
torch.save(optimizer_G.state_dict(), optimizer_g_fname)
if config['use_gan']:
fname = 'checkpoint_D_%d.ckpt' % (ep)
fname = os.path.join(save_path, fname)
save_weights(discriminator, fname, distributed)
optimizer_d_fname = os.path.join(
save_path, 'latest_optimizer_d_state.ckpt')
torch.save(optimizer_D.state_dict(), optimizer_d_fname)
def main():
args = args_parser()
update_config(args.cfg)
if cfg.BASIC.SHOW_CFG:
pprint.pprint(cfg)
# prepare running environment for the whole project
prepare_env(cfg)
# log
writer = SummaryWriter(
log_dir=os.path.join(cfg.BASIC.CKPT_DIR, cfg.BASIC.LOG_DIR))
# dataloader
train_dset = WtalDataset(cfg, cfg.DATASET.TRAIN_SPLIT)
train_loader = DataLoader(train_dset,
batch_size=cfg.TRAIN.BATCH_SIZE,
shuffle=True,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
val_dset = WtalDataset(cfg, cfg.DATASET.VAL_SPLIT)
val_loader = DataLoader(val_dset,
batch_size=cfg.TEST.BATCH_SIZE,
shuffle=False,
num_workers=cfg.BASIC.WORKERS,
pin_memory=cfg.BASIC.PIN_MEMORY)
# network
model = LocNet(cfg)
# model.apply(weight_init)
model.cuda()
# weight_file = "/disk3/zt/code/actloc/thumos/17_CAS_CAM_fast_tuning/output/20class_seed_0_save_model/checkpoint_best_cas_0.2701.pth"
weight_file = '/disk3/zt/code/actloc/thumos/20_0.2701_try/output/debug_save_epoch30/checkpoint_best_cas.pth'
from utils.utils import load_weights
model = load_weights(model, weight_file)
# optimizer
optimizer = optim.Adam(model.parameters(),
lr=cfg.TRAIN.LR,
betas=cfg.TRAIN.BETAS,
weight_decay=cfg.TRAIN.WEIGHT_DECAY)
optimizer.load_state_dict(torch.load(weight_file)['optimizer'])
# criterion
criterion = BasNetLoss()
for epoch in range(1, cfg.TRAIN.EPOCH_NUM + 1):
print('Epoch: %d:' % epoch)
loss_average_cas, loss_average_cam, loss_average_consistency, loss_average_norm, loss_average_cas_inv, loss_average_cam_inv = train(
cfg, train_loader, model, optimizer, criterion)
writer.add_scalar('train_loss/cas', loss_average_cas, epoch)
writer.add_scalar('train_loss/cam', loss_average_cam, epoch)
writer.add_scalar('train_loss/consistency', loss_average_consistency,
epoch)
writer.add_scalar('train_loss/norm', loss_average_norm, epoch)
writer.add_scalar('train_loss/cas_inv', loss_average_cas_inv, epoch)
writer.add_scalar('train_loss/cam_inv', loss_average_cam_inv, epoch)
if cfg.BASIC.VERBOSE:
print(
'loss: cas %f, cam %f, consistency %f, norm %f, cas_inv %f, cam_inv %f'
% (loss_average_cas, loss_average_cam,
loss_average_consistency, loss_average_norm,
loss_average_cas_inv, loss_average_cam_inv))
# decay learning rate
if epoch in cfg.TRAIN.LR_DECAY_EPOCHS:
decay_lr(optimizer, factor=cfg.TRAIN.LR_DECAY_FACTOR)
if epoch % cfg.TEST.EVAL_INTERVAL == 0:
_, _, test_acc_cas, test_acc_cam = evaluate(
cfg, val_loader, model, epoch)
if cfg.BASIC.VERBOSE:
print('test_acc, cas %f, cam %f' %
(test_acc_cas, test_acc_cam))
writer.add_scalar('test_acc/cas', test_acc_cas, epoch)
writer.add_scalar('test_acc/cam', test_acc_cam, epoch)
writer.close()