def generate_iad_files(lfd_params,
model,
dataset_mode,
verbose=False,
backbone=None):
# Create DataLoaders
#assert lfd_params.input_dtype in ["video"], "ERROR: run_classification.py: input_dtype must be 'video'"
#if lfd_params.input_dtype == "video":
from datasets.dataset_video import DatasetVideo as CustomDataset
dataset = CustomDataset(lfd_params,
lfd_params.application.file_directory,
dataset_mode,
verbose=True,
num_segments=lfd_params.input_frames)
data_loader = create_dataloader(dataset,
lfd_params,
dataset_mode,
shuffle=False)
# put model on GPU
net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda()
net.eval()
for i, data_packet in enumerate(data_loader):
obs, label, filename = data_packet
# compute output
iad = net(obs)
iad = iad.detach().cpu().numpy()
for n, file in enumerate(filename):
# format new save name
save_id = file.split('/')
file_id = save_id[-1] + ".npz"
save_id = save_id[:save_id.index("frames")] + [
"iad_" + backbone
] + save_id[save_id.index("frames") + 1:-1]
save_id = '/' + os.path.join(*save_id)
# create a directory to save the ITRs in
if not os.path.exists(save_id):
os.makedirs(save_id)
save_id = os.path.join(save_id, file_id)
if verbose:
print("n: {0}, filename: {1}, saved_id: {2}".format(
n, file, save_id))
# save ITR to file with given name
print(save_id)
print("iad.shape:", iad[n].shape)
np.savez(save_id, data=iad[n])
def train_pipeline(lfd_params, model):
# Create DataLoaders
dataset = CustomDataset(lfd_params,
lfd_params.application.file_directory,
"train",
num_segments=lfd_params.input_frames)
data_loader = create_dataloader(dataset,
lfd_params,
"train",
shuffle=False)
# put model on GPU
#model.use_pipeline = False
#net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda()
#net.eval()
# record mask and threshold values
mask_and_threshold = DITRL_MaskFinder()
for i, data_packet in enumerate(data_loader):
activation_map, label = data_packet
activation_map = activation_map.detach().cpu().numpy()
#print("activation_map:", activation_map.shape)
# compute output
#activation_map = net(obs).detach().cpu().numpy()
for iad in activation_map:
mask_and_threshold.add_data(iad)
mask, threshold = mask_and_threshold.gen_mask_and_threshold()
#model.use_pipeline = True
model.pipeline.pipeline.preprocessing = False
model.pipeline.pipeline.mask_idx = mask
model.pipeline.pipeline.threshold_values = threshold
'''
for i, data_packet in enumerate(data_loader):
activation_map, label = data_packet
activation_map, label = data_packet
activation_map = activation_map.detach().cpu().numpy()
# compute output
#_ = net(obs)
model.pipeline.fit_pipeline()
model.pipeline.pipeline.preprocessing = True
model.pipeline.pipeline.is_training = False
'''
return model
def evaluate_action_trace(lfd_params,
model,
mode="evaluation",
verbose=False,
input_dtype="video",
ablation=False):
# Create DataLoaders
assert input_dtype in [
"video", "iad", "gcn"
], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'"
if input_dtype == "video":
from datasets.dataset_video_trace import DatasetVideoTrace as CustomDataset
elif input_dtype == "iad":
from datasets.dataset_iad_trace import DatasetIADTrace as CustomDataset
#elif input_dtype == "itr":
# from obsolete_files.dataset_itr_trace import DatasetITRTrace as CustomDataset
else:
from datasets.dataset_gcn_trace import DatasetGCNTrace as CustomDataset
dataset = CustomDataset(lfd_params,
lfd_params.application.file_directory,
mode,
eval=True,
trace_path=lfd_params.application.trace_file,
verbose=True,
backbone=lfd_params.model.model_id,
num_segments=lfd_params.input_frames,
ablation=ablation)
data_loader = create_dataloader(dataset, lfd_params, mode, shuffle=False)
# put model on GPU
net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda()
net.eval()
# Train Network
expected_label_list = []
predicted_label_list = []
obs_filename_list = []
with torch.no_grad():
for i, data_packet in enumerate(data_loader):
obs, act, obs_filenames, _ = data_packet
predicted_action_history = []
for j in range(1, act.shape[1] + 1):
o = obs[:, :j]
a = act[:, :j]
# obtain label
label = a[:, -1]
label = torch.argmax(label, dim=1)
# prepare a_history
a_history = np.zeros((1, (len(predicted_action_history) + 1),
NUM_TOTAL_ACTIONS))
for k in range(len(predicted_action_history)):
a_history[0, k, predicted_action_history[k]] = 1
a_history = torch.from_numpy(a_history)
o = o[:, -WIN_HIST:]
a_history = a_history[:, -WIN_HIST:]
# compute output
logits = net(o.float(), a_history.float())
# get label information
expected_label = label.cpu().detach().numpy()
predicted_label = np.argmax(logits.cpu().detach().numpy(),
axis=1)
predicted_action_history.append(predicted_label)
# add data to lists to be returned
act = act.cpu().detach().numpy()
for j in range(act.shape[1]):
if len(expected_label_list) <= j:
expected_label_list.append([])
predicted_label_list.append([])
obs_filename_list.append([])
expected_label_list[j].append(np.argmax(act[0, j]))
predicted_label_list[j].append(predicted_action_history[j][0])
obs_filename_list[j].append(obs_filenames[j][0])
if verbose:
print("file: {:3d}/{:3d}".format(i, len(data_loader)))
print("expected_label:", expected_label)
print("predicted_label:", predicted_label)
print("logits:")
print(logits.cpu().detach().numpy())
df_dict = {}
for i in range(len(expected_label_list)):
df_dict["expected_label_" + str(i)] = expected_label_list[i]
df_dict["predicted_label_" + str(i)] = predicted_label_list[i]
df_dict["obs_filename_" + str(i)] = obs_filename_list[i]
# return Pandas dataframe
return pd.DataFrame(df_dict)
def evaluate_single_action(lfd_params,
model,
mode="evaluation",
verbose=False,
input_dtype="video"):
# Create DataLoaders
assert input_dtype in [
"video", "iad", "gcn"
], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'"
if input_dtype == "video":
from datasets.dataset_video_trace import DatasetVideoTrace as CustomDataset
elif input_dtype == "iad":
from datasets.dataset_iad_trace import DatasetIADTrace as CustomDataset
#elif input_dtype == "itr":
# from obsolete_files.dataset_itr_trace import DatasetITRTrace as CustomDataset
else:
from datasets.dataset_gcn_trace import DatasetGCNTrace as CustomDataset
dataset = CustomDataset(lfd_params,
lfd_params.application.file_directory,
mode,
trace_path=lfd_params.application.trace_file,
verbose=True,
backbone=model.backbone_id,
num_segments=lfd_params.args.num_segments)
data_loader = create_dataloader(dataset, lfd_params, mode, shuffle=False)
# put model on GPU
net = torch.nn.DataParallel(model, device_ids=lfd_params.args.gpus).cuda()
net.eval()
# Train Network
expected_label_list = []
predicted_label_list = []
obs_filename_list = []
trace_id = []
with torch.no_grad():
for i, data_packet in enumerate(data_loader):
obs, act, obs_filenames, _ = data_packet
for j in range(1, act.shape[1]):
o = obs[:, :j]
a = act[:, :j]
# constrain size to a history of 5 timesteps
o = o[:, -WIN_HIST:]
a = a[:, -WIN_HIST:]
# obtain label
label = a[:, -1]
label = torch.argmax(label, dim=1)
# hide label
a[:, -1] = 0
# compute output
logits = net(o.float(), a.float())
# get label information
expected_label = label.cpu().detach().numpy()
predicted_label = np.argmax(logits.cpu().detach().numpy(),
axis=1)
# add data to lists to be returned
expected_label_list.append(expected_label)
predicted_label_list.append(predicted_label)
obs_filename_list.append(obs_filenames[j - 1])
trace_id.append(i)
if verbose:
print("file: {:3d}/{:3d}".format(i, len(data_loader)))
print("expected_label:", expected_label)
print("predicted_label:", predicted_label)
print("logits:")
print(logits.cpu().detach().numpy())
# return Pandas dataframe
return pd.DataFrame({
"expected_label": expected_label_list,
"predicted_label": predicted_label_list,
"obs_filename_list": obs_filename_list,
"trace_id": trace_id,
})
def train(lfd_params,
model,
verbose=False,
input_dtype="video",
ablation=False):
# Create DataLoaders
assert input_dtype in [
"video", "iad", "gcn"
], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'"
if input_dtype == "video":
from datasets.dataset_video_trace import DatasetVideoTrace as CustomDataset
elif input_dtype == "iad":
from datasets.dataset_iad_trace import DatasetIADTrace as CustomDataset
#elif input_dtype == "itr":
# from obsolete_files.dataset_itr_trace import DatasetITRTrace as CustomDataset
else:
from datasets.dataset_gcn_trace import DatasetGCNTrace as CustomDataset
dataset = CustomDataset(lfd_params,
lfd_params.application.file_directory,
"train",
eval=False,
trace_path=lfd_params.application.trace_file,
verbose=True,
backbone=lfd_params.model.model_id,
num_segments=lfd_params.input_frames,
ablation=ablation)
data_loader = create_dataloader(dataset, lfd_params, "train", shuffle=True)
# put model on GPU
params = list(model.parameters())
net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda()
net.train()
# define loss function
criterion = torch.nn.CrossEntropyLoss().cuda()
# define optimizer
optimizer = torch.optim.Adam(params, lr=lfd_params.lr)
# Train Network
loss_record = []
with torch.autograd.detect_anomaly():
epoch = lfd_params.epochs
for e in range(epoch):
cumulative_loss = 0
for i, data_packet in enumerate(data_loader):
#print("i: {:d}/{:d}".format(i, len(data_loader)))
obs, act, obs_filename, act_filename = data_packet
# constrain size to a history of 5 timesteps
obs = obs[:, -WIN_HIST:]
act = act[:, -WIN_HIST:]
# obtain label
label = act[:, -1]
label = torch.argmax(label, dim=1)
# hide label
act[:, -1] = 0
# compute output
logits = net(obs.float(), act.float())
# get loss
loss = criterion(logits, label.long().cuda())
loss.backward()
# optimize SGD
optimizer.step()
optimizer.zero_grad()
if verbose: # and i % 100 == 0:
print("epoch: {:3d}/{:3d}".format(e, epoch))
print("loss:", loss.cpu().detach().numpy())
print("expected:", label.cpu().detach().numpy())
print("pred:",
np.argmax(logits.cpu().detach().numpy(), axis=1))
print("logits:")
print(logits.cpu().detach().numpy())
cumulative_loss += loss.cpu().detach().numpy()
print("e:", e, "loss:", cumulative_loss)
loss_record.append(cumulative_loss)
# save trained model parameters
#model.save_model()
# show loss over time, output placed in Log Directory
import matplotlib.pyplot as plt
plt.plot(loss_record)
# add bells and whistles to plt
plt.title(model.filename)
plt.ylabel("loss")
plt.tight_layout()
# make sure log_dir exists
log_dir = model.filename
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# save plt to file
fig_filename = os.path.join(log_dir, "train_loss.png")
plt.savefig(fig_filename)
# clear plt so I don't draw on top of my multiple images.
plt.clf()
return model
def evaluate(lfd_params,
model,
mode="evaluation",
verbose=False,
input_dtype="video"):
# Create DataLoaders
assert input_dtype in [
"video", "iad", "gcn"
], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'"
if input_dtype == "video":
from datasets.dataset_video import DatasetVideo as CustomDataset
elif input_dtype == "iad":
from datasets.dataset_iad import DatasetIAD as CustomDataset
#elif input_dtype == "itr":
# from obsolete_files.dataset_itr import DatasetITR as CustomDataset
else:
from datasets.dataset_gcn import DatasetGCN as CustomDataset
dataset = CustomDataset(lfd_params,
lfd_params.application.file_directory,
mode,
verbose=True,
num_segments=lfd_params.input_frames,
backbone=lfd_params.model.model_id)
data_loader = create_dataloader(dataset, lfd_params, mode, shuffle=False)
# put model on GPU
net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda()
net.eval()
# Train Network
expected_label_list = []
predicted_label_list = []
filename_list = []
for i, data_packet in enumerate(data_loader):
obs, label, filename = data_packet
obs = obs.float()
# compute output
logits = net(obs)
# get label information
expected_label = label.cpu().detach().numpy()[0]
predicted_label = np.argmax(logits.cpu().detach().numpy(), axis=1)[0]
# add data to lists to be returned
expected_label_list.append(expected_label)
predicted_label_list.append(predicted_label)
filename_list.append(filename)
if verbose:
print("file: {:3d}/{:3d}".format(i, len(data_loader)))
print("expected_label:", expected_label)
print("predicted_label:", predicted_label)
print("logits:")
print(logits.cpu().detach().numpy())
# return Pandas dataframe
return pd.DataFrame({
"expected_label": expected_label_list,
"predicted_label": predicted_label_list,
"filename": filename_list,
})
def generate_itr_files_gcn(lfd_params,
model,
dataset_mode,
verbose=False,
backbone="tsm"):
from torch_geometric.data import Data
# Create DataLoaders
#assert lfd_params.input_dtype in ["video", "itr"], "ERROR: run_videos.py: input_dtype must be 'video' or 'itr'"
#if lfd_params.input_dtype == "video":
# from datasets.dataset_video import DatasetVideo as CustomDataset
#print("itr_gcn")
#print("lfd_params.file_directory:", lfd_params.file_directory)
#print("lfd_params.input_frames:", lfd_params.input_frames)
dataset = CustomDataset(lfd_params,
lfd_params.application.file_directory,
dataset_mode,
verbose=True,
num_segments=lfd_params.input_frames)
data_loader = create_dataloader(dataset,
lfd_params,
dataset_mode,
shuffle=False)
# put model on GPU
net = torch.nn.DataParallel(model, device_ids=lfd_params.gpus).cuda()
net.eval()
for i, data_packet in enumerate(data_loader):
obs, label, filename = data_packet
# compute output
x = net(obs)
node_x, edge_idx, edge_attr = x
for n, file in enumerate(filename):
# format new save name
save_id = file.split('/')
file_id = save_id[-1] # + ".npz"
#print("save_id:", save_id)
save_id = save_id[:save_id.index(
"iad_" + lfd_params.model.model_id)] + [
"gcn_" + backbone
] + save_id[save_id.index("iad_" + lfd_params.model.model_id) +
1:-1]
#print("save_id2:", save_id)
save_id = '/' + os.path.join(*save_id)
# create a directory to save the ITRs in
if not os.path.exists(save_id):
os.makedirs(save_id)
save_id = os.path.join(save_id, file_id)
if verbose:
print("n: {0}, filename: {1}, saved_id: {2}".format(
n, file, save_id))
# save ITR to file with given name
#print("node_x[0].shape:", node_x[0].shape)
#print("save_id_Q:", save_id)
np.savez(save_id,
x=node_x[0],
edge_idx=edge_idx[0],
edge_attr=edge_attr[0])
