def get_video_transform(data_name, split_name, opt): normalizer = video_transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) t_list = [] if split_name == 'train': t_list = [ video_transforms.RandomResizedCrop(opt.crop_size), video_transforms.RandomHorizontalFlip(), video_transforms.ColorJitter(brightness=0.1, contrast=0.1, hue=0.1) ] else: t_list = [ video_transforms.Resize(256), video_transforms.CenterCrop(opt.crop_size) ] t_end = [video_transforms.ToTensor(), normalizer] transform = video_transforms.Compose(t_list + t_end) return transform
def __init__(self, opt, train=True): super(VideoDataset, self).__init__(opt, train=train) # Dataroot & Transform if opt.dataset == 'mgif': data_root = './dataset/moving-gif' vtrans = [vtransforms.Scale(size=128)] elif opt.dataset == 'kth': data_root = './dataset/kth_action/' vtrans = [ vtransforms.CenterCrop(size=120), vtransforms.Scale(size=128) ] elif opt.dataset == 'penn': data_root = './dataset/penn_action/' vtrans = [vtransforms.Scale(size=128)] if self.train: vtrans += [vtransforms.RandomHorizontalFlip()] vtrans += [vtransforms.RandomRotation()] vtrans += [vtransforms.ToTensor(scale=True)] vtrans += [vtransforms.Normalize(0.5, 0.5)] if opt.input_norm else [] self.vtrans = T.Compose(vtrans) if self.train: self.image_path = os.path.join(data_root, 'train') else: self.image_path = os.path.join(data_root, 'test') threshold = self.window_size if opt.irregular else self.sample_size if opt.dataset in ['kth', 'sintel', 'ucf101', 'penn']: self.image_list = os.listdir(self.image_path) elif opt.dataset in ['mgif', 'stickman']: self.image_list = remove_files_under_sample_size( image_path=self.image_path, threshold=threshold) self.image_list = sorted(self.image_list)
assert CONFIG["RGB_I3D_LOAD_MODEL_PATH"] or CONFIG[ "FLOW_I3D_LOAD_MODEL_PATH"] # Setup Dataset and Dataloader if CONFIG["DATASET"] == "original": Dataset = bbdb_dataset.OriginalBBDBDataset elif CONFIG["DATASET"] == "binary": Dataset = bbdb_dataset.BinaryBBDBDataset else: assert False with open("data_split.min.json", "r") as fp: data_split = json.load(fp) test_transforms = transforms.Compose([ video_transforms.Resize(256), video_transforms.CenterCrop(224), ]) dataset = Dataset(segment_filepaths=data_split["test"], segment_length=CONFIG["SEGMENT_LENGTH"], frameskip=CONFIG["FRAMESKIP"], transform=test_transforms) dataloader = DataLoader(dataset, batch_size=CONFIG["BATCH_SIZE"], pin_memory=True) # Setup I3D # TODO(seungjaeryanlee): Allow choosing both if CONFIG["RGB_I3D_LOAD_MODEL_PATH"]: rgb_i3d = InceptionI3d(400, in_channels=3) rgb_i3d.replace_logits(dataset.NUM_LABELS) rgb_i3d.load_state_dict(torch.load(CONFIG["RGB_I3D_LOAD_MODEL_PATH"]))
def main(): global args, best_prec1 args = parser.parse_args() # create model print("Building model ... ") model = build_model() print("Model %s is loaded. " % (args.modality + "_" + args.arch)) if not os.path.exists(args.resume): os.makedirs(args.resume) print("Saving everything to directory %s." % (args.resume)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) cudnn.benchmark = True # Data transforming clip_mean = [0.485, 0.456, 0.406] * args.new_length clip_std = [0.229, 0.224, 0.225] * args.new_length normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) if args.modality == "rgb": scale_ratios = [1.0, 0.875, 0.75, 0.66] elif args.modality == "flow": scale_ratios = [1.0, 0.875, 0.75] else: print("No such modality. Only rgb and flow supported.") train_transform = video_transforms.Compose([ video_transforms.Scale((256)), video_transforms.MultiScaleCrop((224, 224), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor(), normalize, ]) val_transform = video_transforms.Compose([ video_transforms.Scale((256)), video_transforms.CenterCrop((224)), video_transforms.ToTensor(), normalize, ]) # data loading train_setting_file = "train_%s_split%d.txt" % (args.modality, args.split) train_split_file = os.path.join(args.settings, args.dataset, train_setting_file) val_setting_file = "val_%s_split%d.txt" % (args.modality, args.split) val_split_file = os.path.join(args.settings, args.dataset, val_setting_file) if not os.path.exists(train_split_file) or not os.path.exists( val_split_file): print( "No split file exists in %s directory. Preprocess the dataset first" % (args.settings)) train_dataset = datasets.__dict__[args.dataset]( args.data, train_split_file, "train", args.new_length, video_transform=train_transform) val_dataset = datasets.__dict__[args.dataset]( args.data, val_split_file, "val", args.new_length, video_transform=val_transform) print('{} samples found, {} train samples and {} test samples.'.format( len(val_dataset) + len(train_dataset), len(train_dataset), len(val_dataset))) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set prec1 = validate(val_loader, model, criterion) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) if (epoch + 1) % args.save_freq == 0: checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar") save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint_name, args.resume)
def main(): global args, best_prec1 args = parser.parse_args() print(args.modality + " network trained whith the split " + str(args.split) + ".") # create model print("Building model ... ") exits_model, model = build_model(int(args.start_epoch)) if not exits_model: return else: print("Model %s is loaded. " % (args.arch)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # create file where we allocate the models by each args.save_freq epochs if not os.path.exists(args.resume): os.makedirs(args.resume) print("Saving everything to directory %s." % (args.resume)) cudnn.benchmark = True # Data transforming if args.modality == "rgb" or args.modality == "rhythm" or args.modality == "history": is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.485, 0.456, 0.406] * args.new_length clip_std = [0.299, 0.224, 0.225] * args.new_length elif args.modality == "flow": is_color = False scale_ratios = [1.0, 0.875, 0.75] clip_mean = [0.5, 0.5] * args.new_length clip_std = [0.226, 0.226] * args.new_length else: print("No such modality. Only rgb and flow supported.") new_size = 299 if args.arch == 'rgb_inception_v3' else 224 normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) train_transform = video_transforms.Compose([ #video_transforms.Scale((256)), video_transforms.MultiScaleCrop((new_size, new_size), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor(), normalize, ]) val_transform = video_transforms.Compose([ # video_transforms.Scale((256)), video_transforms.CenterCrop((new_size)), video_transforms.ToTensor(), normalize, ]) #createNewDataset("train_%s_split%d.txt" , "new_train.txt") #createNewDataset("val_%s_split%d.txt", "new_test.txt") # data loading #train_setting_file = 'new_train.txt' modality_ = "rgb" if (args.modality == "rhythm" or args.modality == "history") else args.modality train_setting_file = "train_%s_split%d.txt" % (modality_, args.split) train_split_file = os.path.join(args.settings, args.dataset, train_setting_file) #val_setting_file = 'new_test.txt' val_setting_file = "val_%s_split%d.txt" % (modality_, args.split) val_split_file = os.path.join(args.settings, args.dataset, val_setting_file) if not os.path.exists(train_split_file) or not os.path.exists( val_split_file): print( "No split file exists in %s directory. Preprocess the dataset first" % (args.settings)) train_dataset = datasets.__dict__['dataset']( root=args.data, source=train_split_file, phase="train", modality=args.modality, is_color=is_color, new_length=args.new_length, new_width=args.new_width, new_height=args.new_height, video_transform=train_transform) val_dataset = datasets.__dict__['dataset'](root=args.data, source=val_split_file, phase="val", modality=args.modality, is_color=is_color, new_length=args.new_length, new_width=args.new_width, new_height=args.new_height, video_transform=val_transform) print('{} samples found, {} train samples and {} test samples.'.format( len(val_dataset) + len(train_dataset), len(train_dataset), len(val_dataset))) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set prec1 = 0.0 if (epoch + 1) % args.save_freq == 0: prec1 = validate(val_loader, model, criterion) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) if (epoch + 1) % args.save_freq == 0: checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint_" + args.modality + "_split_" + str(args.split) + ".pth.tar") save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint_name, args.resume)
import shutil import math import numpy as np from PIL import Image from tsn_dataset import TSNDataSet from p3d_model import P3D199,get_optim_policies import video_transforms from tsn_models import TSN from torch.nn.utils import clip_grad_norm val_transform=video_transforms.Compose( [ video_transforms.Resize((182,242)), video_transforms.CenterCrop(160), video_transforms.ToTensor(), video_transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225))] ) val_loader=torch.utils.data.DataLoader( TSNDataSet("","tsntest_01.lst", num_segments=2, new_length=16, modality="RGB", image_tmpl="frame{:06d}.jpg", transform=val_transform, random_shift=False), batch_size=1, shuffle=False,
def main(): global args, best_prec1, model, writer, best_loss, length, width, height, input_size, scheduler args = parser.parse_args() training_continue = args.contine if '3D' in args.arch: if 'I3D' in args.arch or 'MFNET3D' in args.arch: if '112' in args.arch: scale = 0.5 else: scale = 1 else: if '224' in args.arch: scale = 1 else: scale = 0.5 elif 'r2plus1d' in args.arch: scale = 0.5 else: scale = 1 print('scale: %.1f' % (scale)) input_size = int(224 * scale) width = int(340 * scale) height = int(256 * scale) saveLocation = "./checkpoint/" + args.dataset + "_" + args.arch + "_split" + str( args.split) if not os.path.exists(saveLocation): os.makedirs(saveLocation) writer = SummaryWriter(saveLocation) # create model if args.evaluate: print("Building validation model ... ") model = build_model_validate() optimizer = AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) elif training_continue: model, startEpoch, optimizer, best_prec1 = build_model_continue() for param_group in optimizer.param_groups: lr = param_group['lr'] #param_group['lr'] = lr print( "Continuing with best precision: %.3f and start epoch %d and lr: %f" % (best_prec1, startEpoch, lr)) else: print("Building model with ADAMW... ") model = build_model() optimizer = AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) startEpoch = 0 if HALF: model.half() # convert to half precision for layer in model.modules(): if isinstance(layer, nn.BatchNorm2d): layer.float() print("Model %s is loaded. " % (args.arch)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() criterion2 = nn.MSELoss().cuda() scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5, verbose=True) print("Saving everything to directory %s." % (saveLocation)) if args.dataset == 'ucf101': dataset = './datasets/ucf101_frames' elif args.dataset == 'hmdb51': dataset = './datasets/hmdb51_frames' elif args.dataset == 'smtV2': dataset = './datasets/smtV2_frames' elif args.dataset == 'window': dataset = './datasets/window_frames' elif args.dataset == 'haa500_basketball': dataset = './datasets/haa500_basketball_frames' else: print("No convenient dataset entered, exiting....") return 0 cudnn.benchmark = True modality = args.arch.split('_')[0] if "3D" in args.arch or 'tsm' in args.arch or 'slowfast' in args.arch or 'r2plus1d' in args.arch: if '64f' in args.arch: length = 64 elif '32f' in args.arch: length = 32 else: length = 16 else: length = 1 # Data transforming if modality == "rgb" or modality == "pose": is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] if 'I3D' in args.arch: if 'resnet' in args.arch: clip_mean = [0.45, 0.45, 0.45] * args.num_seg * length clip_std = [0.225, 0.225, 0.225] * args.num_seg * length else: clip_mean = [0.5, 0.5, 0.5] * args.num_seg * length clip_std = [0.5, 0.5, 0.5] * args.num_seg * length #clip_std = [0.25, 0.25, 0.25] * args.num_seg * length elif 'MFNET3D' in args.arch: clip_mean = [0.48627451, 0.45882353, 0.40784314 ] * args.num_seg * length clip_std = [0.234, 0.234, 0.234] * args.num_seg * length elif "3D" in args.arch: clip_mean = [114.7748, 107.7354, 99.4750] * args.num_seg * length clip_std = [1, 1, 1] * args.num_seg * length elif "r2plus1d" in args.arch: clip_mean = [0.43216, 0.394666, 0.37645] * args.num_seg * length clip_std = [0.22803, 0.22145, 0.216989] * args.num_seg * length elif "rep_flow" in args.arch: clip_mean = [0.5, 0.5, 0.5] * args.num_seg * length clip_std = [0.5, 0.5, 0.5] * args.num_seg * length elif "slowfast" in args.arch: clip_mean = [0.45, 0.45, 0.45] * args.num_seg * length clip_std = [0.225, 0.225, 0.225] * args.num_seg * length else: clip_mean = [0.485, 0.456, 0.406] * args.num_seg * length clip_std = [0.229, 0.224, 0.225] * args.num_seg * length elif modality == "pose": is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.485, 0.456, 0.406] * args.num_seg clip_std = [0.229, 0.224, 0.225] * args.num_seg elif modality == "flow": is_color = False scale_ratios = [1.0, 0.875, 0.75, 0.66] if 'I3D' in args.arch: clip_mean = [0.5, 0.5] * args.num_seg * length clip_std = [0.5, 0.5] * args.num_seg * length elif "3D" in args.arch: clip_mean = [127.5, 127.5] * args.num_seg * length clip_std = [1, 1] * args.num_seg * length else: clip_mean = [0.5, 0.5] * args.num_seg * length clip_std = [0.226, 0.226] * args.num_seg * length elif modality == "both": is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.485, 0.456, 0.406, 0.5, 0.5] * args.num_seg * length clip_std = [0.229, 0.224, 0.225, 0.226, 0.226] * args.num_seg * length else: print("No such modality. Only rgb and flow supported.") normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) if "3D" in args.arch and not ('I3D' in args.arch): train_transform = video_transforms.Compose([ video_transforms.MultiScaleCrop((input_size, input_size), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor2(), normalize, ]) val_transform = video_transforms.Compose([ video_transforms.CenterCrop((input_size)), video_transforms.ToTensor2(), normalize, ]) else: train_transform = video_transforms.Compose([ video_transforms.MultiScaleCrop((input_size, input_size), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor(), normalize, ]) val_transform = video_transforms.Compose([ video_transforms.CenterCrop((input_size)), video_transforms.ToTensor(), normalize, ]) # data loading train_setting_file = "train_%s_split%d.txt" % (modality, args.split) train_split_file = os.path.join(args.settings, args.dataset, train_setting_file) val_setting_file = "val_%s_split%d.txt" % (modality, args.split) val_split_file = os.path.join(args.settings, args.dataset, val_setting_file) if not os.path.exists(train_split_file) or not os.path.exists( val_split_file): print( "No split file exists in %s directory. Preprocess the dataset first" % (args.settings)) train_dataset = datasets.__dict__[args.dataset]( root=dataset, source=train_split_file, phase="train", modality=modality, is_color=is_color, new_length=length, new_width=width, new_height=height, video_transform=train_transform, num_segments=args.num_seg) val_dataset = datasets.__dict__[args.dataset]( root=dataset, source=val_split_file, phase="val", modality=modality, is_color=is_color, new_length=length, new_width=width, new_height=height, video_transform=val_transform, num_segments=args.num_seg) print('{} samples found, {} train samples and {} test samples.'.format( len(val_dataset) + len(train_dataset), len(train_dataset), len(val_dataset))) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: prec1, prec3, _ = validate(val_loader, model, criterion, criterion2, modality) return for epoch in range(startEpoch, args.epochs): # if learning_rate_index > max_learning_rate_decay_count: # break # adjust_learning_rate(optimizer, epoch) train(train_loader, model, criterion, criterion2, optimizer, epoch, modality) # evaluate on validation set prec1 = 0.0 lossClassification = 0 if (epoch + 1) % args.save_freq == 0: prec1, prec3, lossClassification = validate( val_loader, model, criterion, criterion2, modality) writer.add_scalar('data/top1_validation', prec1, epoch) writer.add_scalar('data/top3_validation', prec3, epoch) writer.add_scalar('data/classification_loss_validation', lossClassification, epoch) scheduler.step(lossClassification) # remember best prec@1 and save checkpoint is_best = prec1 >= best_prec1 best_prec1 = max(prec1, best_prec1) # best_in_existing_learning_rate = max(prec1, best_in_existing_learning_rate) # # if best_in_existing_learning_rate > prec1 + 1: # learning_rate_index = learning_rate_index # best_in_existing_learning_rate = 0 if (epoch + 1) % args.save_freq == 0: checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar") if is_best: print("Model works well") save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'best_loss': best_loss, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint_name, saveLocation) checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar") save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'best_loss': best_loss, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint_name, saveLocation) writer.export_scalars_to_json("./all_scalars.json") writer.close()
def flow_test(param_model): model = param_model video_list = os.listdir(video) f = open(video + "flow_result.txt", 'w') for file in video_list: if file.endswith("mp4"): f.write(file + "\n") # file_data = OrderedDict() frame_count = 0 clip_mean = [0.5, 0.5] * 10 clip_std = [0.226, 0.226] * 10 normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) # config the transform to match the network's format transform = video_transforms.Compose([ # video_transforms.Scale((256)), video_transforms.CenterCrop((224)), video_transforms.ToTensor(), normalize, ]) # prepare the translation dictionary label-action data_handler = UCF101_splitter( os.getcwd() + '/datasets/ucf101_splits/', None) data_handler.get_action_index() class_to_idx = data_handler.action_label idx_to_class = {v: k for k, v in class_to_idx.items()} # Start looping on frames received from webcam vs = cv2.VideoCapture(video + file) softmax = torch.nn.Softmax() nn_output = torch.FloatTensor(2 * 10, 224, 224) count = 0 idx = 0 temp = '' x = [] sampled_list = [] while (vs.isOpened()): ret, image = vs.read() if ret is False: break else: image = cv2.resize(image, (342, 256), interpolation=cv2.INTER_LINEAR) x.append(temp) if count == 11: sampled_list = [] # input_var = torch.autograd.Variable(clip_input, volatile=True) temp = '' input_var = clip_input.view(1, 20, 224, 224).cuda() output = model(input_var) output = softmax(output) output = output.data.cpu().numpy() preds = output.argsort()[0][-5:][::-1] pred_classes = [(idx_to_class[str(pred + 1)], output[0, pred]) for pred in preds] value = 0 for i in range(5): if pred_classes[i][0] == label: value = pred_classes[i][1] temp += '{} - {:.2f}\n'.format( pred_classes[i][0], pred_classes[i][1]) f.write(str(value) + "\n") nn_output = torch.FloatTensor(2 * 10, 224, 224) count = 1 if count == 0: old_frame = image.copy() prev = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY) else: next = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback( prev, next, 1, 0.5, 3, 15, 3, 5, 1.2, 0) horz = cv2.normalize(flow[..., 0], None, 0, 255, cv2.NORM_MINMAX) vert = cv2.normalize(flow[..., 1], None, 0, 255, cv2.NORM_MINMAX) horz = horz.astype('uint8') vert = vert.astype('uint8') imgH = Image.fromarray(horz) imgV = Image.fromarray(vert) sampled_list.append(np.expand_dims(imgH, 2)) sampled_list.append(np.expand_dims(imgV, 2)) clip_input = np.concatenate(sampled_list, axis=2) clip_input = transform(clip_input) clip_input = clip_input.float().cuda(async=True) imgH.close() imgV.close() prev = next.copy() count += 1 idx += 1 f.write("----\n") # file_data[file] = flow_value # with open('flow.json', 'w', encoding="utf-8") as make_file: # json.dump(file_data, make_file, ensure_ascii=False, indent="\t") print(idx) vs.release() f.close()
def main(args): global best_prec1, best_loss input_size = int(224 * args.scale) width = int(340 * args.scale) height = int(256 * args.scale) if not os.path.exists(args.savelocation): os.makedirs(args.savelocation) now = time.time() savelocation = os.path.join(args.savelocation, str(now)) os.makedirs(savelocation) logging.basicConfig(filename=os.path.join(savelocation, "log.log"), level=logging.INFO) model = build_model(args.arch, args.pre, args.num_seg, args.resume) optimizer = AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) criterion = nn.CrossEntropyLoss().cuda() criterion2 = nn.MSELoss().cuda() scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5, verbose=True) # if args.dataset=='sign': # dataset="/data/AUTSL/train_img_c" # elif args.dataset=="signd": # dataset="/data/AUTSL/train_img_c" # elif args.dataset=="customd": # dataset="/data/AUTSL/train_img_c" # else: # print("no dataset") # return 0 cudnn.benchmark = True length = 64 scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.43216, 0.394666, 0.37645] * args.num_seg * length clip_std = [0.22803, 0.22145, 0.216989] * args.num_seg * length normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) train_transform = video_transforms.Compose([ video_transforms.CenterCrop(input_size), video_transforms.ToTensor2(), normalize, ]) val_transform = video_transforms.Compose([ video_transforms.CenterCrop((input_size)), video_transforms.ToTensor2(), normalize, ]) # test_transform = video_transforms.Compose([ # video_transforms.CenterCrop((input_size)), # video_transforms.ToTensor2(), # normalize, # ]) # test_file = os.path.join(args.datasetpath, args.testlist) if not os.path.exists(args.trainlist) or not os.path.exists(args.vallist): print( "No split file exists in %s directory. Preprocess the dataset first" % (args.datasetpath)) train_dataset = datasets.__dict__[args.dataset]( root=args.datasetpath, source=args.trainlist, phase="train", modality="rgb", is_color=True, new_length=length, new_width=width, new_height=height, video_transform=train_transform, num_segments=args.num_seg) val_dataset = datasets.__dict__[args.dataset]( root=args.datasetpath, source=args.vallist, phase="val", modality="rgb", is_color=True, new_length=length, new_width=width, new_height=height, video_transform=val_transform, num_segments=args.num_seg) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) best_prec1 = 0 for epoch in range(0, args.epochs): train(length, input_size, train_loader, model, criterion, criterion2, optimizer, epoch) if (epoch + 1) % args.save_freq == 0: is_best = False prec1, prec3, lossClassification = validate( length, input_size, val_loader, model, criterion, criterion2) scheduler.step(lossClassification) if prec1 >= best_prec1: is_best = True best_prec1 = prec1 checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar") text = "save checkpoint {}".format(checkpoint_name) print(text) logging.info(text) save_checkpoint( { "epoch": epoch + 1, "arch": args.arch, "state_dict": model.state_dict(), "prec1": prec1, "optimizer": optimizer.state_dict() }, is_best, checkpoint_name, savelocation)
def main(): global args, best_prec1 args = parser.parse_args() # create model print("Building model ... ") model = build_model() if torch.cuda.is_available(): model = torch.nn.DataParallel(model) print("Model %s is loaded. " % (args.arch)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if not os.path.exists(args.resume): os.makedirs(args.resume) print("Saving everything to directory %s." % (args.resume)) cudnn.benchmark = True # Data transforming if args.modality == "rgb": is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.485, 0.456, 0.406] * args.new_length clip_std = [0.229, 0.224, 0.225] * args.new_length elif args.modality == "flow": is_color = False scale_ratios = [1.0, 0.875, 0.75] clip_mean = [0.5, 0.5] * args.new_length clip_std = [0.226, 0.226] * args.new_length else: print("No such modality. Only rgb and flow supported.") normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) train_transform = video_transforms.Compose([ # video_transforms.Scale((256)), video_transforms.MultiScaleCrop((224, 224), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor(), normalize, ]) val_transform = video_transforms.Compose([ # video_transforms.Scale((256)), video_transforms.CenterCrop((224)), video_transforms.ToTensor(), normalize, ]) # data loading # train_setting_file = "train_%s_split%d.txt" % (args.modality, args.split) # train_split_file = os.path.join(args.settings, args.dataset, train_setting_file) # val_setting_file = "val_%s_split%d.txt" % (args.modality, args.split) # val_split_file = os.path.join(args.settings, args.dataset, val_setting_file) # if not os.path.exists(train_split_file) or not os.path.exists(val_split_file): # print("No split file exists in %s directory. Preprocess the dataset first" % (args.settings)) train_split_file = './datasets/settings/train_set_detail.csv' val_split_file = './datasets/settings/val_set_detail.csv' train_dataset = datasets.__dict__[args.dataset]( root=args.data, #neet to change source=train_split_file, phase="train", modality=args.modality, is_color=is_color, new_length=args.new_length, new_width=args.new_width, new_height=args.new_height, video_transform=train_transform, name_pattern="frame%06d.jpg") # frame000001 val_dataset = datasets.__dict__[args.dataset]( root=args.data, source=val_split_file, phase="val", modality=args.modality, is_color=is_color, new_length=args.new_length, new_width=args.new_width, new_height=args.new_height, video_transform=val_transform, name_pattern="frame%06d.jpg") print('{} samples found, {} train samples and {} test samples.'.format( len(val_dataset) + len(train_dataset), len(train_dataset), len(val_dataset))) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return model_path = '/home/thl/Desktop/challeng/checkpoints/Mulity_100step_900epoch_batch80/model_best.pth.tar' params = torch.load(model_path) model.load_state_dict(params['state_dict']) for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set prec1 = 0.0 if (epoch + 1) % args.save_freq == 0: prec1 = validate(val_loader, model, criterion) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) if (epoch + 1) % args.save_freq == 0: checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar") save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint_name, args.resume)
def rgb_test(param_model): model = param_model f = open(video + "rgb_result.txt", 'w') video_list = os.listdir(video) for file in video_list: if file.endswith("mp4"): f.write(file + "\n") frame_count = 2 clip_mean = [0.485, 0.456, 0.406] * 1 clip_std = [0.229, 0.224, 0.225] * 1 normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) # config the transform to match the network's format transform = video_transforms.Compose([ # video_transforms.Scale((256)), video_transforms.CenterCrop((224)), video_transforms.ToTensor(), normalize, ]) # prepare the translation dictionary label-action data_handler = UCF101_splitter( os.getcwd() + '/datasets/ucf101_splits/', None) data_handler.get_action_index() class_to_idx = data_handler.action_label idx_to_class = {v: k for k, v in class_to_idx.items()} # Start looping on frames received from webcam vs = cv2.VideoCapture(video + file) softmax = torch.nn.Softmax() nn_output = torch.tensor(np.zeros((1, 23)), dtype=torch.float32).cuda() sampled_list = [] first_count = 0 while True: # read each frame and prepare it for feedforward in nn (resize and type) ret, orig_frame = vs.read() if ret is False: break else: orig_frame = cv2.resize(orig_frame, (342, 256), interpolation=cv2.INTER_LINEAR) frame = cv2.cvtColor(orig_frame, cv2.COLOR_BGR2RGB) frame = transform(frame).view(1, 3, 224, 224).cuda() frame = frame.float().cuda(async=True) # feed the frame to the neural network nn_output = model(frame) # vote for class with 25 consecutive frames if frame_count % 10 == 0: nn_output = softmax(nn_output) nn_output = nn_output.data.cpu().numpy() preds = nn_output.argsort()[0][-5:][::-1] pred_classes = [(idx_to_class[str(pred + 1)], nn_output[0, pred]) for pred in preds] red = (0, 0, 255) green = (0, 255, 0) blue = (255, 0, 0) white = (255, 255, 255) yellow = (0, 255, 255) cyan = (255, 255, 0) magenta = (255, 0, 255) thickness = 2 center_x = int(342 / 2.0) center_y = int(256 / 2.0) location = (center_x - 170, center_y + 80) fontScale = 1.5 font = cv2.FONT_HERSHEY_SIMPLEX y0, dy = 180, 20 value = 0 for i in range(5): y = y0 + i * dy if pred_classes[i][0] == label: value = pred_classes[i][1] # reset the process f.write(str(value) + "\n") nn_output = torch.tensor(np.zeros((1, 23)), dtype=torch.float32).cuda() # cv2.imwrite("temp/" + str(frame_count) + ".jpg", orig_frame) # Display the resulting frame and the classified action frame_count += 1 # When everything done, release the capture f.write("----\n") vs.release() f.close()
def main(): global best_prec1 # create model print("Building model ... ") print("Building model ... ", file = f_log) model = build_model(resume_path = args.resume) print("Model %s is loaded. " % (args.arch)) print("Model %s is loaded. " % (args.arch), file = f_log) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) cudnn.benchmark = True # Data transforming if args.modality == "rgb": is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.485, 0.456, 0.406] clip_std = [0.229, 0.224, 0.225] elif args.modality == "tvl1_flow" or args.modality == "lk_flow": is_color = False scale_ratios = [1.0, 0.875, 0.75] clip_mean = [0.5, 0.5] clip_std = [0.226, 0.226] else: print("No such modality. Only rgb and flow supported.") print("No such modality. Only rgb and flow supported.", file = f_log) normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) train_transform = video_transforms.Compose([ #video_transforms.Scale((288)), video_transforms.MultiScaleCrop((256, 256), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor(), normalize, ]) val_transform = video_transforms.Compose([ #video_transforms.Scale((288)), video_transforms.CenterCrop((256)), video_transforms.ToTensor(), normalize, ]) # data loading train_setting_file = "train_%s_split%d.txt" % (args.modality, args.split) train_split_file = os.path.join(args.settings, args.dataset, train_setting_file) val_setting_file = "val_%s_split%d.txt" % (args.modality, args.split) val_split_file = os.path.join(args.settings, args.dataset, val_setting_file) if not os.path.exists(train_split_file) or not os.path.exists(val_split_file): print("No split file exists in %s directory. Preprocess the dataset first" % (args.settings)) print("No split file exists in %s directory. Preprocess the dataset first" % (args.settings), file = f_log ) train_dataset = datasets.__dict__[args.dataset](setting=train_split_file, root=args.data, train=True, new_width=args.new_width, new_height=args.new_height, new_length=args.new_length, target_width=args.new_width, target_height=args.new_height, modality=args.modality, num_segments=args.num_segments, transform=train_transform, name_pattern='frame%06d.jpg') val_dataset = datasets.__dict__[args.dataset](setting=val_split_file, root=args.data, train=False, new_width=args.new_width, new_height=args.new_height, new_length=args.new_length, target_width=args.new_width, target_height=args.new_height, modality=args.modality, num_segments=args.num_segments, transform=val_transform, name_pattern='frame%06d.jpg') print('{} samples found, {} train samples and {} test samples.'.format(len(val_dataset)+len(train_dataset), len(train_dataset), len(val_dataset))) print('{} samples found, {} train samples and {} test samples.'.format(len(val_dataset)+len(train_dataset), len(train_dataset), len(val_dataset)), file = f_log ) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) #, pin_memory=True) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers) #, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch print("start epoch ", epoch) train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set prec1 = 0.0 if (epoch + 1) % args.save_freq == 0: prec1 = validate(val_loader, model, criterion) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) if (epoch + 1) % args.save_freq == 0: checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint.pth.tar") save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint_name, args.save_path)
def main(): global args, prec_list prec_list = [] args = parser.parse_args() full_path = logging(args) print(args.modality + " network trained with the split " + str(args.split) + ".") # create model print("Building model ... ") exits_model, model = build_model(int(args.start_epoch), args.pretrain_weights) if not exits_model: return else: print("Model %s is loaded. " % (args.arch)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) cudnn.benchmark = True # Data transforming if args.modality == "rgb" or args.modality == "rgb2": is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.485, 0.456, 0.406] * args.new_length clip_std = [0.229, 0.224, 0.225] * args.new_length elif args.modality == "flow" or args.modality == "rhythm": is_color = False scale_ratios = [1.0, 0.875, 0.75] clip_mean = [0.5, 0.5] * args.new_length clip_std = [0.226, 0.226] * args.new_length else: print("No such modality. Only rgb and flow supported.") new_size = 299 if args.arch.find("inception_v3") > 0 else 224 normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) train_transform = video_transforms.Compose([ #video_transforms.Scale((256)), video_transforms.MultiScaleCrop((new_size, new_size), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor(), normalize, ]) if args.es: val_transform = video_transforms.Compose([ # video_transforms.Scale((256)), video_transforms.CenterCrop((new_size)), video_transforms.ToTensor(), normalize, ]) modality_ = "rgb" if (args.modality == "rhythm" or args.modality[:3] == "rgb") else "flow" if args.modality == "rgb2": createNewDataset("train_split%d.txt", "new_train.txt", modality_) #createNewDataset("val_%s_split%d.txt", "new_val.txt",modality_) # data loading train_setting_file = "new_train.txt" if args.modality == "rgb2" else "train_split%d.txt" % ( args.split) train_split_file = os.path.join(args.settings, args.dataset, train_setting_file) if not os.path.exists( train_split_file): # or not os.path.exists(val_split_file): print( "No split file exists in %s directory. Preprocess the dataset first" % (args.settings)) extension = ".png" if args.dataset == "hmdb51" and args.modality == "rhythm" else ".jpg" direction_file = "direction.txt" if args.vr_approach == 3 else "direction_video.txt" direction_path = os.path.join(args.settings, args.dataset, direction_file) train_dataset = datasets.__dict__['dataset']( root=args.data, source=train_split_file, phase="train", modality=args.modality, is_color=is_color, new_length=args.new_length, new_width=args.new_width, new_height=args.new_height, video_transform=train_transform, approach_VR=args.vr_approach, extension=extension, direction_path=direction_path) if args.es: val_setting_file = "val_split%d.txt" % (args.split) val_split_file = os.path.join(args.settings, args.dataset, val_setting_file) if not os.path.exists(val_split_file): print( "No split file exists in %s directory. Preprocess the dataset first" % (args.settings)) val_dataset = datasets.__dict__['dataset']( root=args.data, source=val_split_file, phase="val", modality=args.modality, is_color=is_color, new_length=args.new_length, new_width=args.new_width, new_height=args.new_height, video_transform=val_transform, approach_VR=args.vr_approach, extension=extension, direction_path=direction_path) print('{} samples found, {} train samples and {} validation samples.'. format( len(val_dataset) + len(train_dataset), len(train_dataset), len(val_dataset))) else: print('{} train samples found.'.format(len(train_dataset))) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) if args.es: val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) early_stop = EarlyStopping(verbose=True, log_path=os.path.join( full_path, "early_stopping.json")) is_best = False for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) if args.es: # evaluate on validation set losses = validate(val_loader, model, criterion) is_best = early_stop(losses.avg, epoch) if (epoch + 1) % args.save_freq == 0 or is_best: checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint_" + args.modality + "_split_" + str(args.split) + ".pth.tar") es_val = float('inf') if not args.es else early_stop.val_loss_min save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), 'val_loss_min': es_val }, is_best, checkpoint_name, os.path.join(full_path, "checkpoints")) prec_name = "%03d_%s" % (epoch + 1, "prec_split_" + str(args.split) + ".txt") save_precision(prec_name, os.path.join(full_path, "precision")) if args.es and early_stop.early_stop: break if not args.es: # Final model checkpoint_name = "%03d_%s" % (epoch + 1, "checkpoint_" + args.modality + "_split_" + str(args.split) + ".pth.tar") save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), 'val_loss_min': float('inf') }, True, checkpoint_name, os.path.join(full_path, "checkpoints"))
def main(): global args args = parser.parse_args() if args.dataset == 'ucf101': num_categories = 101 elif args.dataset == 'hmdb51': num_categories = 51 elif args.dataset == 'kinetics': num_categories = 400 else: raise ValueError('Unknown dataset ' + args.dataset) start_frame = 0 model_start_time = time.time() params = torch.load(args.weights) #hard code net = models.__dict__[args.arch](pretrained=False, num_classes=num_categories) net.load_state_dict(params['state_dict']) net.cuda() net.eval() model_end_time = time.time() model_time = model_end_time - model_start_time print("Action recognition model is loaded in %4.4f seconds." % (model_time)) ### if args.modality == "rgb": new_length = 1 is_color = True scale_ratios = [1.0, 0.875, 0.75, 0.66] clip_mean = [0.485, 0.456, 0.406] * new_length clip_std = [0.229, 0.224, 0.225] * new_length elif args.modality == "flow": new_length = 10 is_color = False scale_ratios = [1.0, 0.875, 0.75] clip_mean = [0.5, 0.5] * new_length clip_std = [0.226, 0.226] * new_length normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) val_transform = video_transforms.Compose([ video_transforms.GroupCenterCrop(net.input_size), video_transforms.CenterCrop((224)), video_transforms.ToTensor(), normalize, ]) dataset = datasets.load_clip(root=args.data, source=args.test_list, phase="val", modality=args.modality, is_color=is_color, new_length=new_length, new_width=args.new_width, new_height=args.new_height, video_transform=val_transform) data_loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=args.workers * 2, pin_memory=True) data_gen = enumerate(data_loader) total_num = len(data_loader.dataset) output = [] def eval_video(video_data): i, data, label = video_data num_crop = 1 if args.modality == 'rgb': length = 3 elif args.modality == 'flow': length = 10 elif args.modality == 'RGBDiff': length = 18 else: raise ValueError("Unknown modality " + args.modality) input_var = torch.autograd.Variable(data.view(-1, length, data.size(2), data.size(3)), volatile=True) input_var = input_var.type(torch.FloatTensor).cuda() rst = net(input_var).data.cpu().numpy().copy() return i, rst.reshape( (num_crop, args.test_segments, num_categories)).mean(axis=0).reshape( (args.test_segments, 1, num_categories)), label[0] proc_start_time = time.time() max_num = args.max_num if args.max_num > 0 else len(data_loader.dataset) for i, (data, label) in data_gen: data = data.float().cuda(async=True) label = label.cuda(async=True) input_var = torch.autograd.Variable(data, volatile=True) target_var = torch.autograd.Variable(label, volatile=True) rst = net(input_var).data.cpu().numpy() #avg_pred_fc8 = np.mean(rst, axis=1) # print(avg_spatial_pred_fc8.shape) #result_list.append(avg_pred_fc8) # avg_spatial_pred = softmax(avg_spatial_pred_fc8) pred_index = np.argmax(rst) #print (label.cpu().numpy()) #print (pred_index) # print(rst) # if i >= max_num: # break # rst = eval_video((i, data, label)) output.append(rst) if label.cpu().numpy()[0] == pred_index: match_count += 1 cnt_time = time.time() - proc_start_time print('video {} done, total {}/{}, average {} sec/video'.format( i, i + 1, total_num, float(cnt_time) / (i + 1))) print(match_count) print(total_num) print("Accuracy is %4.4f" % (float(match_count) / float(total_num))) np.save("{}_sX_{}_{}.npy".format(args.dataset, args.modality, args.arch), np.array(output)) #hard code
width = int(340 * scale) height = int(256 * scale) length = 32 clip_mean = [0.5, 0.5, 0.5] * length clip_std = [0.5, 0.5, 0.5] * length scale_ratios = [1.0, 0.875, 0.75, 0.66] normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std) train_transform = video_transforms.Compose([ video_transforms.MultiScaleCrop((input_size, input_size), scale_ratios), video_transforms.RandomHorizontalFlip(), video_transforms.ToTensor(), normalize, ]) val_transform = video_transforms.Compose([ video_transforms.CenterCrop((input_size)), video_transforms.ToTensor(), normalize, ]) #define dataset and dataloader train_split_file = "./datasets/settings/haa500_instruments/train_rgb_split1.txt" val_split_file = "./datasets/settings/haa500_instruments/val_rgb_split1.txt" dataset_name = "haa500_instruments" dataset_root = "./datasets/haa500_instruments" train_dataset = haa500_instruments(train_split_file=train_split_file, dataset_root=dataset_root, phase="train", new_length=length, new_width=width,