def __init__(self,args,netdata,log):
self.t = 0
self.args = args
self.sumo_cmd = set_sumo(args.gui, args.roadnet, args.max_steps, args.port)
self.log = log
print(self.sumo_cmd)
self.save_path = set_save_path(args.roadnet,args.tsc)
self.saver = Saver(self.save_path)
self.max_steps = args.max_steps
self.green_t = args.green_duration
self.yellow_t = args.yellow_duration
self.red_t = args.red_duration
self.mode = args.mode
self.scale = args.scale
self.port = args.port
self.netdata = netdata
self.tl_ids = self.netdata['inter'].keys()
self.sumo_process = subprocess.Popen(self.sumo_cmd)
self.conn = traci.connect(self.port)
self.netdata = self.update_netdata()
self.vehiclegen = VehicleGen(self.netdata,self.conn,self.mode,self.scale,self.max_steps)
self.Controllers = {str(id):None for id in self.tl_ids}
self.create_controllers(self.args.tsc)
if self.args.tsc == 'dqn' and self.args.tsc == 'test':
self.load_model()
self.v_start_times = {}
self.v_travel_times = {}
self.episode_performance = []
self.set_item_path()
def train(config):
#### set the save and log path ####
# svname = args.name
# if svname is None:
# svname = config['train_dataset_type'] + '_' + config['model']
# # svname += '_' + config['model_args']['encoder']
# # if config['model_args']['classifier'] != 'linear-classifier':
# # svname += '-' + config['model_args']['classifier']
# if args.tag is not None:
# svname += '_' + args.tag
# save_path = os.path.join('./save', svname)
save_path = config['save_path']
utils.set_save_path(save_path)
utils.set_log_path(save_path)
writer = SummaryWriter(os.path.join(config['save_path'], 'tensorboard'))
yaml.dump(config, open(os.path.join(config['save_path'], 'classifier_config.yaml'), 'w'))
#### make datasets ####
# train
train_folder = config['dataset_path'] + config['train_dataset_type'] + "/training/frames"
test_folder = config['dataset_path'] + config['train_dataset_type'] + "/testing/frames"
# Loading dataset
train_dataset_args = config['train_dataset_args']
test_dataset_args = config['test_dataset_args']
train_dataset = VadDataset(train_folder, transforms.Compose([
transforms.ToTensor(),
]), resize_height=train_dataset_args['h'], resize_width=train_dataset_args['w'], time_step=train_dataset_args['t_length'] - 1)
test_dataset = VadDataset(test_folder, transforms.Compose([
transforms.ToTensor(),
]), resize_height=test_dataset_args['h'], resize_width=test_dataset_args['w'], time_step=test_dataset_args['t_length'] - 1)
train_dataloader = DataLoader(train_dataset, batch_size=train_dataset_args['batch_size'],
shuffle=True, num_workers=train_dataset_args['num_workers'], drop_last=True)
test_dataloader = DataLoader(test_dataset, batch_size=test_dataset_args['batch_size'],
shuffle=False, num_workers=test_dataset_args['num_workers'], drop_last=False)
# for test---- prepare labels
labels = scipy.io.loadmat(config['label_path'])
if config['test_dataset_type'] == 'shanghai':
labels = np.expand_dims(labels, 0)
videos = OrderedDict()
videos_list = sorted(glob.glob(os.path.join(test_folder, '*')))
labels_list = []
label_length = 0
psnr_list = {}
for video in sorted(videos_list):
video_name = video.split('/')[-1]
videos[video_name] = {}
videos[video_name]['path'] = video
videos[video_name]['frame'] = glob.glob(os.path.join(video, '*.jpg'))
videos[video_name]['frame'].sort()
videos[video_name]['length'] = len(videos[video_name]['frame'])
labels_list = np.append(labels_list, labels[video_name][0][4:])
label_length += videos[video_name]['length']
psnr_list[video_name] = []
# Model setting
if config['generator'] == 'cycle_generator_convlstm':
ngf = 64
netG = 'resnet_6blocks'
norm = 'instance'
no_dropout = False
init_type = 'normal'
init_gain = 0.02
gpu_ids = []
model = define_G(train_dataset_args['c'], train_dataset_args['c'],
ngf, netG, norm, not no_dropout, init_type, init_gain, gpu_ids)
elif config['generator'] == 'unet':
# TODO
model = UNet(n_channels=train_dataset_args['c']*(train_dataset_args['t_length']-1),
layer_nums=num_unet_layers, output_channel=train_dataset_args['c'])
else:
raise Exception('The generator is not implemented')
# optimizer setting
params_encoder = list(model.parameters())
params_decoder = list(model.parameters())
params = params_encoder + params_decoder
optimizer, lr_scheduler = utils.make_optimizer(
params, config['optimizer'], config['optimizer_args'])
# set loss, different range with the source version, should change
lam_int = 1.0 * 2
lam_gd = 1.0 * 2
# TODO here we use no flow loss
# lam_op = 0 # 2.0
# op_loss = Flow_Loss()
adversarial_loss = Adversarial_Loss()
# # TODO if use adv
# lam_adv = 0.05
# discriminate_loss = Discriminate_Loss()
alpha = 1
l_num = 2
gd_loss = Gradient_Loss(alpha, train_dataset_args['c'])
int_loss = Intensity_Loss(l_num)
# parallel if muti-gpus
if torch.cuda.is_available():
model.cuda()
if config.get('_parallel'):
model = nn.DataParallel(model)
# Training
utils.log('Start train')
max_frame_AUC, max_roi_AUC = 0,0
base_channel_num = train_dataset_args['c'] * (train_dataset_args['t_length'] - 1)
save_epoch = 5 if config['save_epoch'] is None else config['save_epoch']
for epoch in range(config['epochs']):
model.train()
for j, imgs in enumerate(tqdm(train_dataloader, desc='train', leave=False)):
imgs = imgs.cuda()
# input = imgs[:, :-1, ].view(imgs.shape[0], -1, imgs.shape[-2], imgs.shape[-1])
input = imgs[:, :-1, ]
target = imgs[:, -1, ]
outputs = model(input)
optimizer.zero_grad()
g_int_loss = int_loss(outputs, target)
g_gd_loss = gd_loss(outputs, target)
loss = lam_gd * g_gd_loss + lam_int * g_int_loss
loss.backward()
optimizer.step()
lr_scheduler.step()
utils.log('----------------------------------------')
utils.log('Epoch:' + str(epoch + 1))
utils.log('----------------------------------------')
utils.log('Loss: Reconstruction {:.6f}'.format(loss.item()))
# Testing
utils.log('Evaluation of ' + config['test_dataset_type'])
# Save the model
if epoch % save_epoch == 0 or epoch == config['epochs'] - 1:
if not os.path.exists(save_path):
os.makedirs(save_path)
if not os.path.exists(os.path.join(save_path, "models")):
os.makedirs(os.path.join(save_path, "models"))
# TODO
frame_AUC, roi_AUC = evaluate(test_dataloader, model, labels_list, videos, int_loss, config['test_dataset_type'], test_bboxes=config['test_bboxes'],
frame_height = train_dataset_args['h'], frame_width=train_dataset_args['w'],
is_visual=False, mask_labels_path = config['mask_labels_path'], save_path = os.path.join(save_path, "./final"), labels_dict=labels)
torch.save(model.state_dict(), os.path.join(save_path, 'models/model-epoch-{}.pth'.format(epoch)))
else:
frame_AUC, roi_AUC = evaluate(test_dataloader, model, labels_list, videos, int_loss, config['test_dataset_type'], test_bboxes=config['test_bboxes'],
frame_height = train_dataset_args['h'], frame_width=train_dataset_args['w'])
utils.log('The result of ' + config['test_dataset_type'])
utils.log("AUC: {}%, roi AUC: {}%".format(frame_AUC*100, roi_AUC*100))
if frame_AUC > max_frame_AUC:
max_frame_AUC = frame_AUC
# TODO
torch.save(model.state_dict(), os.path.join(save_path, 'models/max-frame_auc-model.pth'))
evaluate(test_dataloader, model, labels_list, videos, int_loss, config['test_dataset_type'], test_bboxes=config['test_bboxes'],
frame_height = train_dataset_args['h'], frame_width=train_dataset_args['w'],
is_visual=True, mask_labels_path = config['mask_labels_path'], save_path = os.path.join(save_path, "./frame_best"), labels_dict=labels)
if roi_AUC > max_roi_AUC:
max_roi_AUC = roi_AUC
torch.save(model.state_dict(), os.path.join(save_path, 'models/max-roi_auc-model.pth'))
evaluate(test_dataloader, model, labels_list, videos, int_loss, config['test_dataset_type'], test_bboxes=config['test_bboxes'],
frame_height = train_dataset_args['h'], frame_width=train_dataset_args['w'],
is_visual=True, mask_labels_path = config['mask_labels_path'], save_path = os.path.join(save_path, "./roi_best"), labels_dict=labels)
utils.log('----------------------------------------')
utils.log('Training is finished')
utils.log('max_frame_AUC: {}, max_roi_AUC: {}'.format(max_frame_AUC, max_roi_AUC))
def main(config_, save_path):
global config, log, writer
config = config_
log, writer = utils.set_save_path(save_path)
with open(os.path.join(save_path, 'config.yaml'), 'w') as f:
yaml.dump(config, f, sort_keys=False)
train_loader, val_loader = make_data_loaders()
if config.get('data_norm') is None:
config['data_norm'] = {
'inp': {
'sub': [0],
'div': [1]
},
'gt': {
'sub': [0],
'div': [1]
}
}
model, optimizer, epoch_start, lr_scheduler = prepare_training()
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
if n_gpus > 1:
model = nn.parallel.DataParallel(model)
epoch_max = config['epoch_max']
epoch_val = config.get('epoch_val')
epoch_save = config.get('epoch_save')
max_val_v = -1e18
timer = utils.Timer()
for epoch in range(epoch_start, epoch_max + 1):
t_epoch_start = timer.t()
log_info = ['epoch {}/{}'.format(epoch, epoch_max)]
writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch)
train_loss = train(train_loader, model, optimizer)
if lr_scheduler is not None:
lr_scheduler.step()
log_info.append('train: loss={:.4f}'.format(train_loss))
writer.add_scalars('loss', {'train': train_loss}, epoch)
if n_gpus > 1:
model_ = model.module
else:
model_ = model
model_spec = config['model']
model_spec['sd'] = model_.state_dict()
optimizer_spec = config['optimizer']
optimizer_spec['sd'] = optimizer.state_dict()
sv_file = {
'model': model_spec,
'optimizer': optimizer_spec,
'epoch': epoch
}
torch.save(sv_file, os.path.join(save_path, 'epoch-last.pth'))
if (epoch_save is not None) and (epoch % epoch_save == 0):
torch.save(sv_file,
os.path.join(save_path, 'epoch-{}.pth'.format(epoch)))
if (epoch_val is not None) and (epoch % epoch_val == 0):
if n_gpus > 1 and (config.get('eval_bsize') is not None):
model_ = model.module
else:
model_ = model
val_res = eval_psnr(val_loader,
model_,
data_norm=config['data_norm'],
eval_type=config.get('eval_type'),
eval_bsize=config.get('eval_bsize'))
log_info.append('val: psnr={:.4f}'.format(val_res))
writer.add_scalars('psnr', {'val': val_res}, epoch)
if val_res > max_val_v:
max_val_v = val_res
torch.save(sv_file, os.path.join(save_path, 'epoch-best.pth'))
t = timer.t()
prog = (epoch - epoch_start + 1) / (epoch_max - epoch_start + 1)
t_epoch = utils.time_text(t - t_epoch_start)
t_elapsed, t_all = utils.time_text(t), utils.time_text(t / prog)
log_info.append('{} {}/{}'.format(t_epoch, t_elapsed, t_all))
log(', '.join(log_info))
writer.flush()
def train(config):
#### set the save and log path ####
save_path = config['save_path']
utils.set_save_path(save_path)
utils.set_log_path(save_path)
writer = SummaryWriter(os.path.join(config['save_path'], 'tensorboard'))
yaml.dump(config, open(os.path.join(config['save_path'], 'classifier_config.yaml'), 'w'))
device = torch.device('cuda:' + args.gpu)
#### make datasets ####
# train
train_folder = config['dataset_path'] + config['train_dataset_type'] + "/training/frames"
test_folder = config['dataset_path'] + config['train_dataset_type'] + "/testing/frames"
# Loading dataset
train_dataset_args = config['train_dataset_args']
test_dataset_args = config['test_dataset_args']
train_dataset = VadDataset(args,video_folder= train_folder, bbox_folder = config['train_bboxes_path'], flow_folder=config['train_flow_path'],
transform=transforms.Compose([transforms.ToTensor()]),
resize_height=train_dataset_args['h'], resize_width=train_dataset_args['w'],
dataset=config['train_dataset_type'], time_step=train_dataset_args['t_length'] - 1,
device=device)
test_dataset = VadDataset(args,video_folder= test_folder, bbox_folder = config['test_bboxes_path'], flow_folder=config['test_flow_path'],
transform=transforms.Compose([transforms.ToTensor()]),
resize_height=train_dataset_args['h'], resize_width=train_dataset_args['w'],
dataset=config['train_dataset_type'], time_step=train_dataset_args['t_length'] - 1,
device=device)
train_dataloader = DataLoader(train_dataset, batch_size=train_dataset_args['batch_size'],
shuffle=True, num_workers=train_dataset_args['num_workers'], drop_last=True)
test_dataloader = DataLoader(test_dataset, batch_size=test_dataset_args['batch_size'],
shuffle=False, num_workers=test_dataset_args['num_workers'], drop_last=False)
# for test---- prepare labels
labels = np.load('./data/frame_labels_' + config['test_dataset_type'] + '.npy')
if config['test_dataset_type'] == 'shanghai':
labels = np.expand_dims(labels, 0)
videos = OrderedDict()
videos_list = sorted(glob.glob(os.path.join(test_folder, '*')))
labels_list = []
label_length = 0
psnr_list = {}
for video in sorted(videos_list):
video_name = video.split('/')[-1]
videos[video_name] = {}
videos[video_name]['path'] = video
videos[video_name]['frame'] = glob.glob(os.path.join(video, '*.jpg'))
videos[video_name]['frame'].sort()
videos[video_name]['length'] = len(videos[video_name]['frame'])
labels_list = np.append(labels_list, labels[0][4 + label_length:videos[video_name]['length'] + label_length])
label_length += videos[video_name]['length']
psnr_list[video_name] = []
# Model setting
num_unet_layers = 4
discriminator_num_filters = [128, 256, 512, 512]
# for gradient loss
alpha = 1
# for int loss
l_num = 2
pretrain = False
if config['generator'] == 'cycle_generator_convlstm':
ngf = 64
netG = 'resnet_6blocks'
norm = 'instance'
no_dropout = False
init_type = 'normal'
init_gain = 0.02
gpu_ids = []
model = define_G(train_dataset_args['c'], train_dataset_args['c'],
ngf, netG, norm, not no_dropout, init_type, init_gain, gpu_ids)
elif config['generator'] == 'unet':
# generator = UNet(n_channels=train_dataset_args['c']*(train_dataset_args['t_length']-1),
# layer_nums=num_unet_layers, output_channel=train_dataset_args['c'])
model = PreAE(train_dataset_args['c'], train_dataset_args['t_length'], **config['model_args'])
else:
raise Exception('The generator is not implemented')
# generator = torch.load('save/avenue_cycle_generator_convlstm_flownet2_0103/generator-epoch-199.pth')
if config['use_D']:
discriminator=PixelDiscriminator(train_dataset_args['c'],discriminator_num_filters,use_norm=False)
optimizer_D = torch.optim.Adam(discriminator.parameters(),lr=0.00002)
# optimizer setting
params_encoder = list(model.parameters())
params_decoder = list(model.parameters())
params = params_encoder + params_decoder
optimizer_G, lr_scheduler = utils.make_optimizer(
params, config['optimizer'], config['optimizer_args'])
# set loss, different range with the source version, should change
lam_int = 1.0 * 2
lam_gd = 1.0 * 2
# TODO here we use no flow loss
# lam_op = 0 # 2.0
# op_loss = Flow_Loss()
adversarial_loss = Adversarial_Loss()
# TODO if use adv
lam_adv = 0.05
discriminate_loss = Discriminate_Loss()
alpha = 1
l_num = 2
gd_loss = Gradient_Loss(alpha, train_dataset_args['c'])
int_loss = Intensity_Loss(l_num)
object_loss = ObjectLoss(device, l_num)
# parallel if muti-gpus
if torch.cuda.is_available():
model.cuda()
if config['use_D']:
discriminator.cuda()
if config.get('_parallel'):
model = nn.DataParallel(model)
if config['use_D']:
discriminator = nn.DataParallel(discriminator)
# Training
utils.log('Start train')
max_frame_AUC, max_roi_AUC = 0,0
base_channel_num = train_dataset_args['c'] * (train_dataset_args['t_length'] - 1)
save_epoch = 5 if config['save_epoch'] is None else config['save_epoch']
for epoch in range(config['epochs']):
model.train()
for j, (imgs, bbox, flow) in enumerate(tqdm(train_dataloader, desc='train', leave=False)):
imgs = imgs.cuda()
flow = flow.cuda()
# input = imgs[:, :-1, ].view(imgs.shape[0], -1, imgs.shape[-2], imgs.shape[-1])
input = imgs[:, :-1, ]
target = imgs[:, -1, ]
outputs = model(input)
if config['use_D']:
g_adv_loss = adversarial_loss(discriminator(outputs))
else:
g_adv_loss = 0
g_object_loss = object_loss(outputs, target, flow, bbox)
# g_int_loss = int_loss(outputs, target)
g_gd_loss = gd_loss(outputs, target)
g_loss = lam_adv * g_adv_loss + lam_gd * g_gd_loss + lam_int * g_object_loss
optimizer_G.zero_grad()
g_loss.backward()
optimizer_G.step()
train_psnr = utils.psnr_error(outputs,target)
# ----------- update optim_D -------
if config['use_D']:
optimizer_D.zero_grad()
d_loss = discriminate_loss(discriminator(target), discriminator(outputs.detach()))
d_loss.backward()
optimizer_D.step()
lr_scheduler.step()
utils.log('----------------------------------------')
utils.log('Epoch:' + str(epoch + 1))
utils.log('----------------------------------------')
utils.log('Loss: Reconstruction {:.6f}'.format(g_loss.item()))
# Testing
utils.log('Evaluation of ' + config['test_dataset_type'])
# Save the model
if epoch % save_epoch == 0 or epoch == config['epochs'] - 1:
if not os.path.exists(save_path):
os.makedirs(save_path)
if not os.path.exists(os.path.join(save_path, "models")):
os.makedirs(os.path.join(save_path, "models"))
# TODO
frame_AUC = ObjectLoss_evaluate(test_dataloader, model, labels_list, videos, dataset=config['test_dataset_type'],device = device,
frame_height = train_dataset_args['h'], frame_width=train_dataset_args['w'],
is_visual=False, mask_labels_path = config['mask_labels_path'], save_path = os.path.join(save_path, "./final"), labels_dict=labels)
torch.save(model.state_dict(), os.path.join(save_path, 'models/model-epoch-{}.pth'.format(epoch)))
if config['use_D']:
torch.save(discriminator.state_dict(), os.path.join(save_path, 'models/discrominator-epoch-{}.pth'.format(epoch)))
else:
frame_AUC = ObjectLoss_evaluate(test_dataloader, model, labels_list, videos, dataset=config['test_dataset_type'],device=device,
frame_height = train_dataset_args['h'], frame_width=train_dataset_args['w'])
utils.log('The result of ' + config['test_dataset_type'])
utils.log("AUC: {}%".format(frame_AUC*100))
if frame_AUC > max_frame_AUC:
max_frame_AUC = frame_AUC
# TODO
torch.save(model.state_dict(), os.path.join(save_path, 'models/max-frame_auc-model.pth'))
if config['use_D']:
torch.save(discriminator.state_dict(), os.path.join(save_path, 'models/discrominator-epoch-{}.pth'.format(epoch)))
# evaluate(test_dataloader, model, labels_list, videos, int_loss, config['test_dataset_type'], test_bboxes=config['test_bboxes'],
# frame_height = train_dataset_args['h'], frame_width=train_dataset_args['w'],
# is_visual=True, mask_labels_path = config['mask_labels_path'], save_path = os.path.join(save_path, "./frame_best"), labels_dict=labels)
utils.log('----------------------------------------')
utils.log('Training is finished')
utils.log('max_frame_AUC: {}'.format(max_frame_AUC))
top1=top1),
file=F_txt)
print(' * Prec@1 {top1.avg:.3f} Best_prec1 {best_prec1:.3f}'.format(
top1=top1, best_prec1=best_prec1))
print(' * Prec@1 {top1.avg:.3f} Best_prec1 {best_prec1:.3f}'.format(
top1=top1, best_prec1=best_prec1),
file=F_txt)
return top1.avg, losses.avg, accuracies
if __name__ == '__main__':
# Save path
opt.outf, F_txt = utils.set_save_path(opt)
# Check if the cuda is available
if torch.cuda.is_available() and not opt.cuda:
print(
"WARNING: You have a CUDA device, so you should probably run with --cuda"
)
# ========================================== Model Config ===============================================
global best_prec1_val, best_prec1_test, epoch_index
best_prec1_val = 0
best_prec1_test = 0
epoch_index = 0
FewShotNet = model_dict[opt.method_name]
def train(config):
#### set the save and log path ####
svname = args.name
if svname is None:
svname = config['train_dataset_type'] + '_' + config[
'generator'] + '_' + config['flow_model']
if args.tag is not None:
svname += '_' + args.tag
save_path = os.path.join('./save', svname)
utils.set_save_path(save_path)
utils.set_log_path(save_path)
writer = SummaryWriter(os.path.join(save_path, 'tensorboard'))
yaml.dump(config,
open(os.path.join(save_path, 'classifier_config.yaml'), 'w'))
#### make datasets ####
# train
train_folder = config['dataset_path'] + config[
'train_dataset_type'] + "/training/frames"
test_folder = config['dataset_path'] + config[
'train_dataset_type'] + "/testing/frames"
# Loading dataset
train_dataset_args = config['train_dataset_args']
test_dataset_args = config['test_dataset_args']
train_dataset = VadDataset(train_folder,
transforms.Compose([
transforms.ToTensor(),
]),
resize_height=train_dataset_args['h'],
resize_width=train_dataset_args['w'],
time_step=train_dataset_args['t_length'] - 1)
test_dataset = VadDataset(test_folder,
transforms.Compose([
transforms.ToTensor(),
]),
resize_height=test_dataset_args['h'],
resize_width=test_dataset_args['w'],
time_step=test_dataset_args['t_length'] - 1)
train_dataloader = DataLoader(
train_dataset,
batch_size=train_dataset_args['batch_size'],
shuffle=True,
num_workers=train_dataset_args['num_workers'],
drop_last=True)
test_dataloader = DataLoader(test_dataset,
batch_size=test_dataset_args['batch_size'],
shuffle=False,
num_workers=test_dataset_args['num_workers'],
drop_last=False)
# for test---- prepare labels
labels = np.load('./data/frame_labels_' + config['test_dataset_type'] +
'.npy')
if config['test_dataset_type'] == 'shanghai':
labels = np.expand_dims(labels, 0)
videos = OrderedDict()
videos_list = sorted(glob.glob(os.path.join(test_folder, '*')))
labels_list = []
label_length = 0
psnr_list = {}
for video in sorted(videos_list):
# video_name = video.split('/')[-1]
# windows
video_name = os.path.split(video)[-1]
videos[video_name] = {}
videos[video_name]['path'] = video
videos[video_name]['frame'] = glob.glob(os.path.join(video, '*.jpg'))
videos[video_name]['frame'].sort()
videos[video_name]['length'] = len(videos[video_name]['frame'])
labels_list = np.append(
labels_list,
labels[0][4 + label_length:videos[video_name]['length'] +
label_length])
label_length += videos[video_name]['length']
psnr_list[video_name] = []
# Model setting
num_unet_layers = 4
discriminator_num_filters = [128, 256, 512, 512]
# for gradient loss
alpha = 1
# for int loss
l_num = 2
pretrain = False
if config['generator'] == 'cycle_generator_convlstm':
ngf = 64
netG = 'resnet_6blocks'
norm = 'instance'
no_dropout = False
init_type = 'normal'
init_gain = 0.02
gpu_ids = []
generator = define_G(train_dataset_args['c'], train_dataset_args['c'],
ngf, netG, norm, not no_dropout, init_type,
init_gain, gpu_ids)
elif config['generator'] == 'unet':
# generator = UNet(n_channels=train_dataset_args['c']*(train_dataset_args['t_length']-1),
# layer_nums=num_unet_layers, output_channel=train_dataset_args['c'])
model = PreAE(train_dataset_args['c'], train_dataset_args['t_length'],
**config['model_args'])
else:
raise Exception('The generator is not implemented')
# generator = torch.load('save/avenue_cycle_generator_convlstm_flownet2_0103/generator-epoch-199.pth')
discriminator = PixelDiscriminator(train_dataset_args['c'],
discriminator_num_filters,
use_norm=False)
# discriminator = torch.load('save/avenue_cycle_generator_convlstm_flownet2_0103/discriminator-epoch-199.pth')
# if not pretrain:
# generator.apply(weights_init_normal)
# discriminator.apply(weights_init_normal)
# if use flownet
# if config['flow_model'] == 'flownet2':
# flownet2SD_model_path = 'flownet2/FlowNet2_checkpoint.pth.tar'
# flow_network = FlowNet2(args).eval()
# flow_network.load_state_dict(torch.load(flownet2SD_model_path)['state_dict'])
# elif config['flow_model'] == 'liteflownet':
# lite_flow_model_path = 'liteFlownet/network-sintel.pytorch'
# flow_network = Network().eval()
# flow_network.load_state_dict(torch.load(lite_flow_model_path))
# different range with the source version, should change
lam_int = 1.0 * 2
lam_gd = 1.0 * 2
# here we use no flow loss
lam_op = 0 # 2.0
lam_adv = 0.05
adversarial_loss = Adversarial_Loss()
discriminate_loss = Discriminate_Loss()
gd_loss = Gradient_Loss(alpha, train_dataset_args['c'])
op_loss = Flow_Loss()
int_loss = Intensity_Loss(l_num)
step = 0
utils.log('initializing the model with Generator-Unet {} layers,'
'PixelDiscriminator with filters {} '.format(
num_unet_layers, discriminator_num_filters))
g_lr = 0.0002
d_lr = 0.00002
optimizer_G = torch.optim.Adam(generator.parameters(), lr=g_lr)
optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=d_lr)
# # optimizer setting
# params_encoder = list(generator.encoder.parameters())
# params_decoder = list(generator.decoder.parameters())
# params = params_encoder + params_decoder
# optimizer, lr_scheduler = utils.make_optimizer(
# params, config['optimizer'], config['optimizer_args'])
#
# loss_func_mse = nn.MSELoss(reduction='none')
# parallel if muti-gpus
if torch.cuda.is_available():
generator.cuda()
discriminator.cuda()
# # if use flownet
# flow_network.cuda()
adversarial_loss.cuda()
discriminate_loss.cuda()
gd_loss.cuda()
op_loss.cuda()
int_loss.cuda()
if config.get('_parallel'):
generator = nn.DataParallel(generator)
discriminator = nn.DataParallel(discriminator)
# if use flownet
# flow_network = nn.DataParallel(flow_network)
adversarial_loss = nn.DataParallel(adversarial_loss)
discriminate_loss = nn.DataParallel(discriminate_loss)
gd_loss = nn.DataParallel(gd_loss)
op_loss = nn.DataParallel(op_loss)
int_loss = nn.DataParallel(int_loss)
# Training
utils.log('Start train')
max_accuracy = 0
base_channel_num = train_dataset_args['c'] * (
train_dataset_args['t_length'] - 1)
save_epoch = 5 if config['save_epoch'] is None else config['save_epoch']
for epoch in range(config['epochs']):
generator.train()
for j, imgs in enumerate(
tqdm(train_dataloader, desc='train', leave=False)):
imgs = imgs.cuda()
input = imgs[:, :-1, ]
input_last = input[:, -1, ]
target = imgs[:, -1, ]
# input = input.view(input.shape[0], -1, input.shape[-2],input.shape[-1])
# only for debug
# input0=imgs[:, 0,]
# input1 = imgs[:, 1, ]
# gt_flow_esti_tensor = torch.cat([input0, input1], 1)
# flow_gt = batch_estimate(gt_flow_esti_tensor, flow_network)[0]
# objectOutput = open('./out_train.flo', 'wb')
# np.array([80, 73, 69, 72], np.uint8).tofile(objectOutput)
# np.array([flow_gt.size(2), flow_gt.size(1)], np.int32).tofile(objectOutput)
# np.array(flow_gt.detach().cpu().numpy().transpose(1, 2, 0), np.float32).tofile(objectOutput)
# objectOutput.close()
# break
# ------- update optim_G --------------
outputs = generator(input)
# pred_flow_tensor = torch.cat([input_last, outputs], 1)
# gt_flow_tensor = torch.cat([input_last, target], 1)
# flow_pred = batch_estimate(pred_flow_tensor, flow_network)
# flow_gt = batch_estimate(gt_flow_tensor, flow_network)
# if you want to use flownet2SD, comment out the part in front
# #### if use flownet ####
# pred_flow_esti_tensor = torch.cat([input_last.view(-1,3,1,input.shape[-2],input.shape[-1]),
# outputs.view(-1,3,1,input.shape[-2],input.shape[-1])], 2)
# gt_flow_esti_tensor = torch.cat([input_last.view(-1,3,1,input.shape[-2],input.shape[-1]),
# target.view(-1,3,1,input.shape[-2],input.shape[-1])], 2)
# flow_gt=flow_network(gt_flow_esti_tensor*255.0)
# flow_pred=flow_network(pred_flow_esti_tensor*255.0)
##############################
# g_op_loss = op_loss(flow_pred, flow_gt) ## flow loss
g_op_loss = 0
g_adv_loss = adversarial_loss(discriminator(outputs))
g_int_loss = int_loss(outputs, target)
g_gd_loss = gd_loss(outputs, target)
g_loss = lam_adv * g_adv_loss + lam_gd * g_gd_loss + lam_op * g_op_loss + lam_int * g_int_loss
optimizer_G.zero_grad()
g_loss.backward()
optimizer_G.step()
train_psnr = utils.psnr_error(outputs, target)
# ----------- update optim_D -------
optimizer_D.zero_grad()
d_loss = discriminate_loss(discriminator(target),
discriminator(outputs.detach()))
d_loss.backward()
optimizer_D.step()
# break
# lr_scheduler.step()
utils.log('----------------------------------------')
utils.log('Epoch:' + str(epoch + 1))
utils.log('----------------------------------------')
utils.log("g_loss: {} d_loss {}".format(g_loss, d_loss))
utils.log('\t gd_loss {}, op_loss {}, int_loss {} ,'.format(
g_gd_loss, g_op_loss, g_int_loss))
utils.log('\t train psnr{}'.format(train_psnr))
# Testing
utils.log('Evaluation of ' + config['test_dataset_type'])
for video in sorted(videos_list):
# video_name = video.split('/')[-1]
video_name = os.path.split(video)[-1]
psnr_list[video_name] = []
generator.eval()
video_num = 0
# label_length += videos[videos_list[video_num].split('/')[-1]]['length']
label_length = videos[os.path.split(
videos_list[video_num])[1]]['length']
for k, imgs in enumerate(
tqdm(test_dataloader, desc='test', leave=False)):
if k == label_length - 4 * (video_num + 1):
video_num += 1
label_length += videos[os.path.split(
videos_list[video_num])[1]]['length']
imgs = imgs.cuda()
input = imgs[:, :-1, ]
target = imgs[:, -1, ]
# input = input.view(input.shape[0], -1, input.shape[-2], input.shape[-1])
outputs = generator(input)
mse_imgs = int_loss((outputs + 1) / 2, (target + 1) / 2).item()
# psnr_list[videos_list[video_num].split('/')[-1]].append(utils.psnr(mse_imgs))
psnr_list[os.path.split(videos_list[video_num])[1]].append(
utils.psnr(mse_imgs))
# Measuring the abnormality score and the AUC
anomaly_score_total_list = []
for video in sorted(videos_list):
# video_name = video.split('/')[-1]
video_name = os.path.split(video)[1]
anomaly_score_total_list += utils.anomaly_score_list(
psnr_list[video_name])
anomaly_score_total_list = np.asarray(anomaly_score_total_list)
accuracy = utils.AUC(anomaly_score_total_list,
np.expand_dims(1 - labels_list, 0))
utils.log('The result of ' + config['test_dataset_type'])
utils.log('AUC: ' + str(accuracy * 100) + '%')
# Save the model
if epoch % save_epoch == 0 or epoch == config['epochs'] - 1:
# torch.save(model, os.path.join(
# save_path, 'model-epoch-{}.pth'.format(epoch)))
torch.save(
generator,
os.path.join(save_path,
'generator-epoch-{}.pth'.format(epoch)))
torch.save(
discriminator,
os.path.join(save_path,
'discriminator-epoch-{}.pth'.format(epoch)))
if accuracy > max_accuracy:
torch.save(generator, os.path.join(save_path, 'generator-max'))
torch.save(discriminator,
os.path.join(save_path, 'discriminator-max'))
utils.log('----------------------------------------')
utils.log('Training is finished')
def main(args):
global log, writer
log, writer = utils.set_save_path(args.output_dir)
log(args)
device = torch.device(args.device)
# fix the seed for reproducibility
seed = args.seed
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
model, criterion, optimizer, lr_scheduler, train_loader, val_loader = prepare_training(
args)
model.to(device)
criterion.to(device)
cudnn.benchmark = True
if args.evaluate:
validate(val_loader, model, criterion, device, 0)
return
# training
output_dir = Path(args.output_dir)
log("Start training")
timer = utils.Timer()
for epoch in range(args.start_epoch, args.epochs + 1):
t_epoch_start = timer.t()
log_info = ['epoch {}/{}'.format(epoch, args.epochs)]
writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch)
train_stats = train(train_loader, model, criterion, optimizer, epoch,
device)
lr_scheduler.step()
log_info.append('train:')
log_info = log_info + [
'{}={:.4f}'.format(k, v.avg) for k, v in train_stats.items()
]
if args.output_dir:
checkpoint_path = output_dir / 'checkpoint.pth'
if epoch > 0 and epoch % args.save_every == 0:
sv_file = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'args': args
}
torch.save(sv_file, checkpoint_path)
test_stats = validate(val_loader, model, criterion, device, epoch)
log_info.append('eval:')
log_info = log_info + [
'{}={:.4f}'.format(k, v.avg) for k, v in test_stats.items()
]
writer.add_scalars('loss', {
'train': train_stats['losses'].avg,
'eval': test_stats['losses'].avg
}, epoch)
writer.add_scalars('Acc', {
'train': train_stats['top1'].avg,
'eval': test_stats['top1'].avg
}, epoch)
writer.add_figure('eval', utils.generate_fig(test_stats), epoch)
writer.add_figure('train', utils.generate_fig(train_stats), epoch)
t = timer.t()
prog = (epoch - args.start_epoch + 1) / (args.epochs -
args.start_epoch + 1)
t_epoch = utils.time_text(t - t_epoch_start)
t_elapsed, t_all = utils.time_text(t), utils.time_text(t / prog)
log_info.append('{} {}/{}'.format(t_epoch, t_elapsed, t_all))
log(', '.join(log_info))
writer.flush()
def train(config):
#### set the save and log path ####
svname = args.name
if svname is None:
svname = 'classifier_{}'.format(config['train_dataset'])
svname += '_' + config['model_args']['encoder']
clsfr = config['model_args']['classifier']
if clsfr != 'linear-classifier':
svname += '-' + clsfr
if args.tag is not None:
svname += '_' + args.tag
save_path = os.path.join('./save', svname)
utils.set_save_path(save_path)
utils.set_log_path(save_path)
writer = SummaryWriter(os.path.join(save_path, 'tensorboard'))
yaml.dump(config, open(os.path.join(save_path, 'config.yaml'), 'w'))
#### make datasets ####
# train
dataloader_train = get_standard_loader(config['dataset_path'],
config['train_dataset'],
**config['train_dataset_args'])
utils.log('train dataset: {}'.format(config['train_dataset']))
# val
if config.get('val_dataset'):
eval_val = True
dataloader_val = get_standard_loader(config['dataset_path'],
config['val_dataset'],
**config['val_dataset_args'])
else:
eval_val = False
# few-shot test
if config.get('test_dataset'):
test_epoch = config.get('test_epoch')
if test_epoch is None:
ef_epoch = 5
test_fs = True
n_way = 5
n_query = 15
n_shots = [1, 5]
dataloaders_test = []
for n_shot in n_shots:
dataloader_test = get_meta_loader(config['dataset_path'],
config['test_dataset'],
ways=n_way,
shots=n_shot,
query_shots=n_query,
**config['test_dataset_args'])
dataloaders_test.append(dataloader_test)
else:
test_fs = False
#### Model and Optimizer ####
if config.get('load'):
model_sv = torch.load(config['load'])
model = models.load(model_sv)
else:
model = models.make(config['model'], **config['model_args'])
if test_fs:
fs_model = models.make('meta-baseline', encoder=None)
fs_model.encoder = model.encoder
if config.get('_parallel'):
model = nn.DataParallel(model)
if test_fs:
fs_model = nn.DataParallel(fs_model)
optimizer, lr_scheduler = utils.make_optimizer(model.parameters(),
config['optimizer'],
**config['optimizer_args'])
#### train and test ####
max_epoch = config['max_epoch']
save_epoch = config.get('save_epoch')
max_va = 0.
for epoch in range(1, max_epoch + 1):
aves_keys = ['tl', 'ta', 'vl', 'va']
if test_fs:
for n_shot in n_shots:
aves_keys += ['fsa-' + str(n_shot)]
# train
model.train()
for data, label in tqdm(dataloader_train, desc='train', leave=False):
data, label = data.cuda(), label.cuda()
logits = model(data)
loss = F.cross_entropy(logits, label)
acc = utils.compute_acc(logits, label)
optimizer.zero_grad()
loss.backward()
optimizer.step()
logits = None
loss = None
# eval
if eval_val:
model.eval()
for data, label in tqdm(dataloader_val, desc='val', leave=False):
data, label = data.cuda(), label.cuda()
with torch.no_grad():
logits = model(data)
loss = F.cross_entropy(logits, label)
acc = utils.compute_acc(logits, label)
if test_fs and (epoch % ef_epoch == 0 or epoch == max_epoch):
fs_model.eval()
for i, n_shot in enumerate(n_shots):
np.random.seed(0)
for data in tqdm(dataloaders_test[i],
desc='fs-' + str(n_shot),
leave=False):
train_inputs, train_targets = data["train"]
train_inputs = train_inputs.cuda()
train_targets = train_targets.cuda()
query_inputs, query_targets = data["test"]
with torch.no_grad():
logits = fs_model(
train_inputs.view(train_inputs.shape[0], n_way,
n_shot,
*train_inputs.shape[-3:]),
query_inputs).view(-1, n_way)
acc = utils.compute_acc(logits, label)
# post
if lr_scheduler is not None:
lr_scheduler.step()
for k, v in aves.items():
aves[k] = v.item()
def main(train_args, save_path):
global log, writer
log, writer = utils.set_save_path(save_path)
train_loader, val_loader = make_data_loaders(train_args, log)
model, optimizer, epoch_start, lr_scheduler = prepare_training(train_args)
gpu_ids = [int(i) for i in train_args.gpu.split(',')]
if len(gpu_ids) > 1:
model = torch.nn.DataParallel(model, device_ids=gpu_ids)
epoch_max = train_args.epochs
epoch_val = train_args.epoch_val
epoch_save = train_args.epoch_save
max_val_v = -1e18
timer = utils.Timer()
for epoch in range(epoch_start, epoch_max + 1):
t_epoch_start = timer.t()
log_info = ['epoch {}/{}'.format(epoch, epoch_max)]
# writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch)
train_loss = train(train_loader, model, optimizer)
if lr_scheduler is not None:
lr_scheduler.step()
log_info.append('train: loss={:.4f}'.format(train_loss))
writer.add_scalars('loss', {'train': train_loss}, epoch)
if len(gpu_ids) > 1:
model_ = model.module
else:
model_ = model
sv_file = {
'model': model_.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch
}
torch.save(sv_file, os.path.join(save_path, 'epoch-last.pth'))
if (epoch_save is not None) and (epoch % epoch_save == 0):
torch.save(sv_file, os.path.join(save_path, 'epoch-{}.pth'.format(epoch)))
if (epoch_val is not None) and (epoch % epoch_val == 0):
if len(gpu_ids) > 1 and (train_args.eval_bsize is not None):
model_ = model.module
else:
model_ = model
val_res = eval_psnr(val_loader, model_, inp_data_norm=train_args.inp_data_norm,
gt_data_norm=train_args.gt_data_norm,
eval_type=train_args.eval_type, eval_bsize=train_args.eval_bsize)
log_info.append('val: psnr={:.4f}'.format(val_res))
writer.add_scalars('psnr', {'val': val_res}, epoch)
if val_res > max_val_v:
max_val_v = val_res
torch.save(sv_file, os.path.join(save_path, 'epoch-best.pth'))
t = timer.t()
prog = (epoch - epoch_start + 1) / (epoch_max - epoch_start + 1)
t_epoch = utils.time_text(t - t_epoch_start)
t_elapsed, t_all = utils.time_text(t), utils.time_text(t / prog)
log_info.append('{} {}/{}'.format(t_epoch, t_elapsed, t_all))
log(', '.join(log_info))
writer.flush()
