def generate_iad_files(lfd_params, model, dataset_mode, verbose=False, backbone=None): # Create DataLoaders #assert lfd_params.input_dtype in ["video"], "ERROR: run_classification.py: input_dtype must be 'video'" #if lfd_params.input_dtype == "video": from datasets.dataset_video import DatasetVideo as CustomDataset dataset = CustomDataset(lfd_params, lfd_params.application.file_directory, dataset_mode, verbose=True, num_segments=lfd_params.input_frames) data_loader = create_dataloader(dataset, lfd_params, dataset_mode, shuffle=False) # put model on GPU net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda() net.eval() for i, data_packet in enumerate(data_loader): obs, label, filename = data_packet # compute output iad = net(obs) iad = iad.detach().cpu().numpy() for n, file in enumerate(filename): # format new save name save_id = file.split('/') file_id = save_id[-1] + ".npz" save_id = save_id[:save_id.index("frames")] + [ "iad_" + backbone ] + save_id[save_id.index("frames") + 1:-1] save_id = '/' + os.path.join(*save_id) # create a directory to save the ITRs in if not os.path.exists(save_id): os.makedirs(save_id) save_id = os.path.join(save_id, file_id) if verbose: print("n: {0}, filename: {1}, saved_id: {2}".format( n, file, save_id)) # save ITR to file with given name print(save_id) print("iad.shape:", iad[n].shape) np.savez(save_id, data=iad[n])
def train_pipeline(lfd_params, model): # Create DataLoaders dataset = CustomDataset(lfd_params, lfd_params.application.file_directory, "train", num_segments=lfd_params.input_frames) data_loader = create_dataloader(dataset, lfd_params, "train", shuffle=False) # put model on GPU #model.use_pipeline = False #net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda() #net.eval() # record mask and threshold values mask_and_threshold = DITRL_MaskFinder() for i, data_packet in enumerate(data_loader): activation_map, label = data_packet activation_map = activation_map.detach().cpu().numpy() #print("activation_map:", activation_map.shape) # compute output #activation_map = net(obs).detach().cpu().numpy() for iad in activation_map: mask_and_threshold.add_data(iad) mask, threshold = mask_and_threshold.gen_mask_and_threshold() #model.use_pipeline = True model.pipeline.pipeline.preprocessing = False model.pipeline.pipeline.mask_idx = mask model.pipeline.pipeline.threshold_values = threshold ''' for i, data_packet in enumerate(data_loader): activation_map, label = data_packet activation_map, label = data_packet activation_map = activation_map.detach().cpu().numpy() # compute output #_ = net(obs) model.pipeline.fit_pipeline() model.pipeline.pipeline.preprocessing = True model.pipeline.pipeline.is_training = False ''' return model
def evaluate_action_trace(lfd_params, model, mode="evaluation", verbose=False, input_dtype="video", ablation=False): # Create DataLoaders assert input_dtype in [ "video", "iad", "gcn" ], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'" if input_dtype == "video": from datasets.dataset_video_trace import DatasetVideoTrace as CustomDataset elif input_dtype == "iad": from datasets.dataset_iad_trace import DatasetIADTrace as CustomDataset #elif input_dtype == "itr": # from obsolete_files.dataset_itr_trace import DatasetITRTrace as CustomDataset else: from datasets.dataset_gcn_trace import DatasetGCNTrace as CustomDataset dataset = CustomDataset(lfd_params, lfd_params.application.file_directory, mode, eval=True, trace_path=lfd_params.application.trace_file, verbose=True, backbone=lfd_params.model.model_id, num_segments=lfd_params.input_frames, ablation=ablation) data_loader = create_dataloader(dataset, lfd_params, mode, shuffle=False) # put model on GPU net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda() net.eval() # Train Network expected_label_list = [] predicted_label_list = [] obs_filename_list = [] with torch.no_grad(): for i, data_packet in enumerate(data_loader): obs, act, obs_filenames, _ = data_packet predicted_action_history = [] for j in range(1, act.shape[1] + 1): o = obs[:, :j] a = act[:, :j] # obtain label label = a[:, -1] label = torch.argmax(label, dim=1) # prepare a_history a_history = np.zeros((1, (len(predicted_action_history) + 1), NUM_TOTAL_ACTIONS)) for k in range(len(predicted_action_history)): a_history[0, k, predicted_action_history[k]] = 1 a_history = torch.from_numpy(a_history) o = o[:, -WIN_HIST:] a_history = a_history[:, -WIN_HIST:] # compute output logits = net(o.float(), a_history.float()) # get label information expected_label = label.cpu().detach().numpy() predicted_label = np.argmax(logits.cpu().detach().numpy(), axis=1) predicted_action_history.append(predicted_label) # add data to lists to be returned act = act.cpu().detach().numpy() for j in range(act.shape[1]): if len(expected_label_list) <= j: expected_label_list.append([]) predicted_label_list.append([]) obs_filename_list.append([]) expected_label_list[j].append(np.argmax(act[0, j])) predicted_label_list[j].append(predicted_action_history[j][0]) obs_filename_list[j].append(obs_filenames[j][0]) if verbose: print("file: {:3d}/{:3d}".format(i, len(data_loader))) print("expected_label:", expected_label) print("predicted_label:", predicted_label) print("logits:") print(logits.cpu().detach().numpy()) df_dict = {} for i in range(len(expected_label_list)): df_dict["expected_label_" + str(i)] = expected_label_list[i] df_dict["predicted_label_" + str(i)] = predicted_label_list[i] df_dict["obs_filename_" + str(i)] = obs_filename_list[i] # return Pandas dataframe return pd.DataFrame(df_dict)
def evaluate_single_action(lfd_params, model, mode="evaluation", verbose=False, input_dtype="video"): # Create DataLoaders assert input_dtype in [ "video", "iad", "gcn" ], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'" if input_dtype == "video": from datasets.dataset_video_trace import DatasetVideoTrace as CustomDataset elif input_dtype == "iad": from datasets.dataset_iad_trace import DatasetIADTrace as CustomDataset #elif input_dtype == "itr": # from obsolete_files.dataset_itr_trace import DatasetITRTrace as CustomDataset else: from datasets.dataset_gcn_trace import DatasetGCNTrace as CustomDataset dataset = CustomDataset(lfd_params, lfd_params.application.file_directory, mode, trace_path=lfd_params.application.trace_file, verbose=True, backbone=model.backbone_id, num_segments=lfd_params.args.num_segments) data_loader = create_dataloader(dataset, lfd_params, mode, shuffle=False) # put model on GPU net = torch.nn.DataParallel(model, device_ids=lfd_params.args.gpus).cuda() net.eval() # Train Network expected_label_list = [] predicted_label_list = [] obs_filename_list = [] trace_id = [] with torch.no_grad(): for i, data_packet in enumerate(data_loader): obs, act, obs_filenames, _ = data_packet for j in range(1, act.shape[1]): o = obs[:, :j] a = act[:, :j] # constrain size to a history of 5 timesteps o = o[:, -WIN_HIST:] a = a[:, -WIN_HIST:] # obtain label label = a[:, -1] label = torch.argmax(label, dim=1) # hide label a[:, -1] = 0 # compute output logits = net(o.float(), a.float()) # get label information expected_label = label.cpu().detach().numpy() predicted_label = np.argmax(logits.cpu().detach().numpy(), axis=1) # add data to lists to be returned expected_label_list.append(expected_label) predicted_label_list.append(predicted_label) obs_filename_list.append(obs_filenames[j - 1]) trace_id.append(i) if verbose: print("file: {:3d}/{:3d}".format(i, len(data_loader))) print("expected_label:", expected_label) print("predicted_label:", predicted_label) print("logits:") print(logits.cpu().detach().numpy()) # return Pandas dataframe return pd.DataFrame({ "expected_label": expected_label_list, "predicted_label": predicted_label_list, "obs_filename_list": obs_filename_list, "trace_id": trace_id, })
def train(lfd_params, model, verbose=False, input_dtype="video", ablation=False): # Create DataLoaders assert input_dtype in [ "video", "iad", "gcn" ], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'" if input_dtype == "video": from datasets.dataset_video_trace import DatasetVideoTrace as CustomDataset elif input_dtype == "iad": from datasets.dataset_iad_trace import DatasetIADTrace as CustomDataset #elif input_dtype == "itr": # from obsolete_files.dataset_itr_trace import DatasetITRTrace as CustomDataset else: from datasets.dataset_gcn_trace import DatasetGCNTrace as CustomDataset dataset = CustomDataset(lfd_params, lfd_params.application.file_directory, "train", eval=False, trace_path=lfd_params.application.trace_file, verbose=True, backbone=lfd_params.model.model_id, num_segments=lfd_params.input_frames, ablation=ablation) data_loader = create_dataloader(dataset, lfd_params, "train", shuffle=True) # put model on GPU params = list(model.parameters()) net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda() net.train() # define loss function criterion = torch.nn.CrossEntropyLoss().cuda() # define optimizer optimizer = torch.optim.Adam(params, lr=lfd_params.lr) # Train Network loss_record = [] with torch.autograd.detect_anomaly(): epoch = lfd_params.epochs for e in range(epoch): cumulative_loss = 0 for i, data_packet in enumerate(data_loader): #print("i: {:d}/{:d}".format(i, len(data_loader))) obs, act, obs_filename, act_filename = data_packet # constrain size to a history of 5 timesteps obs = obs[:, -WIN_HIST:] act = act[:, -WIN_HIST:] # obtain label label = act[:, -1] label = torch.argmax(label, dim=1) # hide label act[:, -1] = 0 # compute output logits = net(obs.float(), act.float()) # get loss loss = criterion(logits, label.long().cuda()) loss.backward() # optimize SGD optimizer.step() optimizer.zero_grad() if verbose: # and i % 100 == 0: print("epoch: {:3d}/{:3d}".format(e, epoch)) print("loss:", loss.cpu().detach().numpy()) print("expected:", label.cpu().detach().numpy()) print("pred:", np.argmax(logits.cpu().detach().numpy(), axis=1)) print("logits:") print(logits.cpu().detach().numpy()) cumulative_loss += loss.cpu().detach().numpy() print("e:", e, "loss:", cumulative_loss) loss_record.append(cumulative_loss) # save trained model parameters #model.save_model() # show loss over time, output placed in Log Directory import matplotlib.pyplot as plt plt.plot(loss_record) # add bells and whistles to plt plt.title(model.filename) plt.ylabel("loss") plt.tight_layout() # make sure log_dir exists log_dir = model.filename if not os.path.exists(log_dir): os.makedirs(log_dir) # save plt to file fig_filename = os.path.join(log_dir, "train_loss.png") plt.savefig(fig_filename) # clear plt so I don't draw on top of my multiple images. plt.clf() return model
def evaluate(lfd_params, model, mode="evaluation", verbose=False, input_dtype="video"): # Create DataLoaders assert input_dtype in [ "video", "iad", "gcn" ], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'" if input_dtype == "video": from datasets.dataset_video import DatasetVideo as CustomDataset elif input_dtype == "iad": from datasets.dataset_iad import DatasetIAD as CustomDataset #elif input_dtype == "itr": # from obsolete_files.dataset_itr import DatasetITR as CustomDataset else: from datasets.dataset_gcn import DatasetGCN as CustomDataset dataset = CustomDataset(lfd_params, lfd_params.application.file_directory, mode, verbose=True, num_segments=lfd_params.input_frames, backbone=lfd_params.model.model_id) data_loader = create_dataloader(dataset, lfd_params, mode, shuffle=False) # put model on GPU net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda() net.eval() # Train Network expected_label_list = [] predicted_label_list = [] filename_list = [] for i, data_packet in enumerate(data_loader): obs, label, filename = data_packet obs = obs.float() # compute output logits = net(obs) # get label information expected_label = label.cpu().detach().numpy()[0] predicted_label = np.argmax(logits.cpu().detach().numpy(), axis=1)[0] # add data to lists to be returned expected_label_list.append(expected_label) predicted_label_list.append(predicted_label) filename_list.append(filename) if verbose: print("file: {:3d}/{:3d}".format(i, len(data_loader))) print("expected_label:", expected_label) print("predicted_label:", predicted_label) print("logits:") print(logits.cpu().detach().numpy()) # return Pandas dataframe return pd.DataFrame({ "expected_label": expected_label_list, "predicted_label": predicted_label_list, "filename": filename_list, })
def generate_itr_files_gcn(lfd_params, model, dataset_mode, verbose=False, backbone="tsm"): from torch_geometric.data import Data # Create DataLoaders #assert lfd_params.input_dtype in ["video", "itr"], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'" #if lfd_params.input_dtype == "video": # from datasets.dataset_video import DatasetVideo as CustomDataset #print("itr_gcn") #print("lfd_params.file_directory:", lfd_params.file_directory) #print("lfd_params.input_frames:", lfd_params.input_frames) dataset = CustomDataset(lfd_params, lfd_params.application.file_directory, dataset_mode, verbose=True, num_segments=lfd_params.input_frames) data_loader = create_dataloader(dataset, lfd_params, dataset_mode, shuffle=False) # put model on GPU net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda() net.eval() for i, data_packet in enumerate(data_loader): obs, label, filename = data_packet # compute output x = net(obs) node_x, edge_idx, edge_attr = x for n, file in enumerate(filename): # format new save name save_id = file.split('/') file_id = save_id[-1] # + ".npz" #print("save_id:", save_id) save_id = save_id[:save_id.index( "iad_" + lfd_params.model.model_id)] + [ "gcn_" + backbone ] + save_id[save_id.index("iad_" + lfd_params.model.model_id) + 1:-1] #print("save_id2:", save_id) save_id = '/' + os.path.join(*save_id) # create a directory to save the ITRs in if not os.path.exists(save_id): os.makedirs(save_id) save_id = os.path.join(save_id, file_id) if verbose: print("n: {0}, filename: {1}, saved_id: {2}".format( n, file, save_id)) # save ITR to file with given name #print("node_x[0].shape:", node_x[0].shape) #print("save_id_Q:", save_id) np.savez(save_id, x=node_x[0], edge_idx=edge_idx[0], edge_attr=edge_attr[0])