def __init__(self,
nuscenes_version: str = 'v1.0-mini',
data_path: str = "data/v1.0-mini",
n_scenes: int = None,
threshold: int = 0.5,
model: Union[str, torch.nn.Module] = None) -> None:
if torch.cuda.is_available():
self.device = torch.device('cuda')
print('Using device: GPU\n')
else:
self.device = torch.device('cpu')
print('Using device: CPU\n')
# init dataset
self.version = nuscenes_version
self.n_scenes = n_scenes
self.nuscenes = create_nuscenes(data_path, nuscenes_version)
self.dataset = NuscenesBEVDataset(nuscenes=self.nuscenes,
n_scenes=n_scenes)
# init model
if isinstance(model, str):
frame_depth, _, _ = self.dataset.grid_size
self.model = Detector(img_depth=frame_depth)
self.model.load_state_dict(torch.load(model))
elif isinstance(model, torch.nn.Module):
self.model = model
self.model.to(self.device)
self.model.train() # keeps dropouts active
self.threshold = threshold
def __init__(
self,
det_model_dir,
emb_model_dir,
use_gpu=False,
run_mode='fluid',
threshold=0.5,
max_cosine_distance=0.2,
nn_budget=100,
max_iou_distance=0.9,
max_age=70,
n_init=3
):
self.detector = Detector(det_model_dir, use_gpu, run_mode)
self.emb = Embedding(emb_model_dir, use_gpu)
self.threshold = threshold
metric = NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
self.tracker = Tracker(metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)
class DeepSort(object):
def __init__(
self,
det_model_dir,
emb_model_dir,
use_gpu=False,
run_mode='fluid',
threshold=0.5,
max_cosine_distance=0.2,
nn_budget=100,
max_iou_distance=0.9,
max_age=70,
n_init=3
):
self.detector = Detector(det_model_dir, use_gpu, run_mode)
self.emb = Embedding(emb_model_dir, use_gpu)
self.threshold = threshold
metric = NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
self.tracker = Tracker(metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)
def update(self, ori_img):
self.height, self.width = ori_img.shape[:2]
results = self.detector.predict(ori_img, self.threshold)
if results is None:
return None
else:
tlwh, xyxy, confidences = results
if not confidences.tolist():
return None
# generate detections
features = self.get_features(xyxy, ori_img)
detections = [Detection(tlwh[i], conf, features[i]) for i,conf in enumerate(confidences)]
# update tracker
self.tracker.predict()
self.tracker.update(detections)
# output bbox identities
outputs = []
for track in self.tracker.tracks:
if not track.is_confirmed() or track.time_since_update > 1:
continue
box = track.to_tlbr()
x1,y1,x2,y2 = box
track_id = track.track_id
outputs.append(np.array([x1,y1,x2,y2,track_id], dtype=np.int))
if len(outputs) > 0:
outputs = np.stack(outputs,axis=0)
return outputs
def get_features(self, xyxy, ori_img):
crops = []
for bbox in xyxy:
crop = ori_img[bbox[1]:bbox[3], bbox[0]:bbox[2], :]
crops.append(crop)
features = self.emb.predict(crops)
return features
def __init__(self,
det_model_dir,
emb_model_dir,
use_gpu=False,
run_mode='fluid',
use_dynamic_shape=False,
trt_min_shape=1,
trt_max_shape=1280,
trt_opt_shape=640,
trt_calib_mode=False,
cpu_threads=1,
enable_mkldnn=False,
threshold=0.5,
max_cosine_distance=0.2,
nn_budget=100,
max_iou_distance=0.9,
max_age=70,
n_init=3):
self.threshold = threshold
self.detector = Detector(model_dir=det_model_dir,
use_gpu=use_gpu,
run_mode=run_mode,
use_dynamic_shape=use_dynamic_shape,
trt_min_shape=trt_min_shape,
trt_max_shape=trt_max_shape,
trt_opt_shape=trt_opt_shape,
trt_calib_mode=trt_calib_mode,
cpu_threads=cpu_threads,
enable_mkldnn=enable_mkldnn)
self.emb = Embedding(emb_model_dir, use_gpu, enable_mkldnn,
cpu_threads)
metric = NearestNeighborDistanceMetric("cosine", max_cosine_distance,
nn_budget)
self.tracker = Tracker(metric,
max_iou_distance=max_iou_distance,
max_age=max_age,
n_init=n_init)
def eval(model_path: str, nuscenes_version: str = 'v1.0-mini', data_path: str = "data/v1.0-mini", n_scenes: int = None,
n_loader_workers: int = 4, batch_size: int = 12):
"""
Evaluate model.
:param model_path: relative path to save model weights
:param nuscenes_version: version of the dataset
:param data_path: relative path to data folder
:param n_scenes: number of scenes in dataset
:param n_loader_workers: number of CPU workers for data loader processing
:param batch_size: batch size
"""
# set up computing device for pytorch
if torch.cuda.is_available():
device = torch.device('cuda')
print('Using device: GPU\n')
else:
device = torch.device('cpu')
print('Using device: CPU\n')
# set up dataset and model
nuscenes = create_nuscenes(data_path, version=nuscenes_version)
eval_dataset = NuscenesBEVDataset(nuscenes=nuscenes, n_scenes=n_scenes, mode='val')
eval_loader = DataLoader(eval_dataset, batch_size=batch_size, shuffle=True, num_workers=n_loader_workers,
collate_fn=frames_bboxes_collate_fn, pin_memory=True)
print('Validation loader is ready.',
f'Number of batches in eval loader: {len(eval_loader)}\n', sep='\n')
frame_depth, frame_width, frame_length = eval_dataset.grid_size
model = Detector(img_depth=frame_depth).to(device)
# load model from checkpoint
model.load_state_dict(torch.load(model_path, map_location='cpu'))
model.to(device)
criterion = DetectionLoss()
detector_out_shape = (batch_size, model.out_channels, frame_width // (2 ** model.n_pools),
frame_length // (2 ** model.n_pools))
gt_former = GroundTruthFormer((frame_width, frame_length), detector_out_shape, device=device)
eval_loss, eval_score = run_epoch(model, eval_loader, criterion, gt_former, epoch=1, mode='val', device=device)
return eval_loss, eval_score
(trainX, testX, trainY, testY) = train_test_split(data, labels,
test_size=0.20)
# construct the training image generator for data augmentation
aug = ImageDataGenerator(
rotation_range=20,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
fill_mode="nearest")
# build the model
print("[INFO] building model...")
model = Detector.build(inputShape=inputShape)
# compile the model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
model.compile(loss="binary_crossentropy", optimizer=opt,
metrics=["accuracy"])
# train the model
print("[INFO] training model...")
H = model.fit(
aug.flow(trainX, trainY, batch_size=BS),
steps_per_epoch=len(trainX) // BS,
validation_data=(testX, testY),
validation_steps=len(testX) // BS,
epochs=EPOCHS)
def main(run_id, pretrained, data_files, model_params, training_params,
device):
best_acc1 = 0
batch_size = training_params['batch_size']
test_batch_size = training_params['test_batch_size']
epochs = training_params['epochs']
start_epoch = training_params['start_epoch']
n_warmup_steps = training_params['n_warmup_steps']
log_interval = training_params['log_interval']
# model is trained for binary classification (for datalaoder)
if model_params['NUM_SPOOF_CLASS'] == 2:
binary_class = True
else:
binary_class = False
kwargs = {
'num_workers': 2,
'pin_memory': True
} if device == torch.device('cuda') else {}
# create model
model = Detector(**model_params).to(device)
num_model_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print('===> Model total parameter: {}'.format(num_model_params))
# Wrap model for multi-GPUs, if necessary
if device == torch.device('cuda') and torch.cuda.device_count() > 1:
print('multi-gpu')
model = nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
optim = optimizer.ScheduledOptim(
torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
betas=(0.9, 0.98),
eps=1e-09,
weight_decay=1e-4,
lr=3e-4,
amsgrad=True), training_params['n_warmup_steps'])
# optionally resume from a checkpoint
if pretrained:
if os.path.isfile(pretrained):
print("===> loading checkpoint '{}'".format(pretrained))
checkpoint = torch.load(pretrained)
start_epoch = checkpoint['epoch']
best_acc1 = checkpoint['best_acc1']
model.load_state_dict(checkpoint['state_dict'])
optim.load_state_dict(checkpoint['optimizer'])
print("===> loaded checkpoint '{}' (epoch {})".format(
pretrained, checkpoint['epoch']))
else:
print("===> no checkpoint found at '{}'".format(pretrained))
# Data loading code
train_data = SpoofDatsetSystemID(data_files['train_scp'],
data_files['train_utt2index'],
binary_class)
val_data = SpoofDatsetSystemID(data_files['dev_scp'],
data_files['dev_utt2index'], binary_class)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
shuffle=True,
**kwargs)
val_loader = torch.utils.data.DataLoader(val_data,
batch_size=test_batch_size,
shuffle=True,
**kwargs)
best_epoch = 0
early_stopping, max_patience = 0, 100 # for early stopping
os.makedirs("model_snapshots/" + run_id, exist_ok=True)
for epoch in range(start_epoch, start_epoch + epochs):
trainer.train(train_loader, model, optim, epoch, device, log_interval)
acc1 = validate.validate(val_loader, data_files['dev_utt2systemID'],
model, device, log_interval)
is_best = acc1 > best_acc1
best_acc1 = max(acc1, best_acc1)
# adjust learning rate + early stopping
if is_best:
early_stopping = 0
best_epoch = epoch + 1
else:
early_stopping += 1
if epoch - best_epoch > 2:
optim.increase_delta()
best_epoch = epoch + 1
if early_stopping == max_patience:
break
# save model
optimizer.save_checkpoint(
{
'epoch': epoch,
'state_dict': model.state_dict(),
'best_acc1': best_acc1,
'optimizer': optim.state_dict(),
}, is_best, "model_snapshots/" + str(run_id),
str(epoch) + ('_%.3f' % acc1) + ".pth.tar")
def main(args):
result_dir_path = Path(args.result_dir)
result_dir_path.mkdir(parents=True, exist_ok=True)
with Path(args.setting).open("r") as f:
setting = json.load(f)
pprint.pprint(setting)
if args.g >= 0 and torch.cuda.is_available():
device = torch.device(f"cuda:{args.g:d}")
print(f"GPU mode: {args.g:d}")
else:
device = torch.device("cpu")
print("CPU mode")
mnist_neg = get_mnist_num(set(setting["label"]["neg"]))
neg_loader = DataLoader(mnist_neg,
batch_size=setting["iterator"]["batch_size"])
generator = get_generator().to(device)
discriminator = get_discriminator().to(device)
opt_g = torch.optim.Adam(
generator.parameters(),
lr=setting["optimizer"]["alpha"],
betas=(setting["optimizer"]["beta1"], setting["optimizer"]["beta2"]),
weight_decay=setting["regularization"]["weight_decay"])
opt_d = torch.optim.Adam(
discriminator.parameters(),
lr=setting["optimizer"]["alpha"],
betas=(setting["optimizer"]["beta1"], setting["optimizer"]["beta2"]),
weight_decay=setting["regularization"]["weight_decay"])
trainer = Engine(
GANTrainer(generator,
discriminator,
opt_g,
opt_d,
device=device,
**setting["updater"]))
# テスト用
test_neg = get_mnist_num(set(setting["label"]["neg"]), train=False)
test_neg_loader = DataLoader(test_neg, setting["iterator"]["batch_size"])
test_pos = get_mnist_num(set(setting["label"]["pos"]), train=False)
test_pos_loader = DataLoader(test_pos, setting["iterator"]["batch_size"])
detector = Detector(generator, discriminator,
setting["updater"]["noise_std"], device).to(device)
log_dict = {}
evaluator = evaluate_accuracy(log_dict, detector, test_neg_loader,
test_pos_loader, device)
plotter = plot_metrics(log_dict, ["accuracy", "precision", "recall", "f"],
"iteration", result_dir_path / "metrics.pdf")
printer = print_logs(log_dict,
["iteration", "accuracy", "precision", "recall", "f"])
img_saver = save_img(generator, test_pos, test_neg,
result_dir_path / "images",
setting["updater"]["noise_std"], device)
trainer.add_event_handler(Events.ITERATION_COMPLETED(every=1000),
evaluator)
trainer.add_event_handler(Events.ITERATION_COMPLETED(every=1000), plotter)
trainer.add_event_handler(Events.ITERATION_COMPLETED(every=1000), printer)
trainer.add_event_handler(Events.ITERATION_COMPLETED(every=1000),
img_saver)
# 指定されたiterationで終了
trainer.add_event_handler(
Events.ITERATION_COMPLETED(once=setting["iteration"]),
lambda engine: engine.terminate())
trainer.run(neg_loader, max_epochs=10**10)
class MCProcessor:
"""
Forms Monte Carlo based uncertanties and visualizes them
:param nuscenes_version: version of the dataset
:param data_path: relative path to data folder
:param n_scenes: number of scenes in dataset
:param threshold: threshold for choosing is bbox or not
:return: Tuple[torch.tensor, np.ndarray] - first - grid tensor, second - gt_bboxes
"""
def __init__(self,
nuscenes_version: str = 'v1.0-mini',
data_path: str = "data/v1.0-mini",
n_scenes: int = None,
threshold: int = 0.5,
model: Union[str, torch.nn.Module] = None) -> None:
if torch.cuda.is_available():
self.device = torch.device('cuda')
print('Using device: GPU\n')
else:
self.device = torch.device('cpu')
print('Using device: CPU\n')
# init dataset
self.version = nuscenes_version
self.n_scenes = n_scenes
self.nuscenes = create_nuscenes(data_path, nuscenes_version)
self.dataset = NuscenesBEVDataset(nuscenes=self.nuscenes,
n_scenes=n_scenes)
# init model
if isinstance(model, str):
frame_depth, _, _ = self.dataset.grid_size
self.model = Detector(img_depth=frame_depth)
self.model.load_state_dict(torch.load(model))
elif isinstance(model, torch.nn.Module):
self.model = model
self.model.to(self.device)
self.model.train() # keeps dropouts active
self.threshold = threshold
def visualise_monte_carlo(self,
sample_id: int = 0,
n_samples: int = 10,
batch_size: int = 4,
save_imgs: bool = False,
saving_folder: str = 'pics') -> None:
"""
Visualize predictions obtained via Monte Carlo estimations and save plots if needed
:param sample_id: number of frame (grid) in the dataset
:param n_samples: number of samples for Monte Carlo approach
:param batch_size: size of batch
:param save_imgs: - flag, if true - save figs to folder pics
:param saving_folder: - path to the folder, where images will be saved (creates new if there is none)
"""
mean_class, _, mean_reg, sigma_reg = self.apply_monte_carlo(
sample_id, n_samples, batch_size)
fig, ax_gt, ax_pred = self._vis_mc(mean_class, mean_reg, sigma_reg,
sample_id)
if save_imgs:
if not os.path.exists(saving_folder):
os.makedirs(saving_folder)
img_path = f'{saving_folder}/{self.version}_{self.n_scenes}_{sample_id}_full.png'
fig.savefig(img_path)
img_path = f'{saving_folder}/{self.version}_{self.n_scenes}_{sample_id}_gt.png'
extent = ax_gt.get_window_extent().transformed(
fig.dpi_scale_trans.inverted())
fig.savefig(img_path, bbox_inches=extent)
img_path = f'{saving_folder}/{self.version}_{self.n_scenes}_{sample_id}_pred.png'
extent_ped = ax_pred.get_window_extent().transformed(
fig.dpi_scale_trans.inverted())
fig.savefig(img_path, bbox_inches=extent_ped)
plt.show()
def apply_monte_carlo(self, sample_id: int = 0, n_samples: int = 10, batch_size: int = 4) \
-> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Apply Monte Carlo dropout for representing model uncertainty
:param sample_id: id of frame (grid) in the dataset
:param n_samples: number of samples for Monte Carlo approach
:param batch_size: batch size
:return: tuple of four torch.Tensors:
1st - mean values of class model prediction
2nd - standard deviations of class model prediction
3rd - mean values of regression model prediction
4th - standard deviations of regression model prediction
"""
assert n_samples > 1, "Need minimum 2 samples to calculate unbiased variance"
grid, boxes = self.dataset[sample_id]
class_output, reg_output = self.model(grid[None].to(self.device))
samples_class, samples_reg = class_output, reg_output
# TODO: append to single batch for speeding up
for i in range(math.ceil((n_samples - 1) / batch_size)):
current_batch_size = min((n_samples - 1) - (i * batch_size),
batch_size)
stacked_grid = torch.stack(current_batch_size * [grid])
class_output, reg_output = self.model(stacked_grid.to(self.device))
samples_class = torch.cat((samples_class, class_output))
samples_reg = torch.cat((reg_output, samples_reg))
# calculate stats from samples set
samples_class, samples_reg = samples_class.detach(
), samples_reg.detach()
mean_reg = torch.mean(samples_reg, dim=0).unsqueeze(0)
variance_reg = torch.var(samples_reg, dim=0)
sigma_reg = torch.sqrt(variance_reg).unsqueeze(0)
mean_class = torch.mean(samples_class, dim=0).unsqueeze(0)
variance_class = torch.var(samples_class, dim=0)
sigma_class = torch.sqrt(variance_class).unsqueeze(0)
return mean_class, sigma_class, mean_reg, sigma_reg
def _vis_mc(self,
mean_class: torch.Tensor,
mean_regr: torch.Tensor,
sigma_regr: torch.Tensor,
sample_id: int = 0) -> Tuple[plt.Figure, plt.Axes, plt.Axes]:
"""
Visualize predictions obtained via Monte Carlo estimations
:param mean_class: mean of classification predictions
:param mean_regr: mean of regression predictions
:param sigma_regr: standard deviation of regression predictions
:param sample_id: number of frame (grid) in the dataset
:return: tuple of three pyplot objects:
1st - figure object
2nd - plot with ground truth bounding boxes
3rd - plt with predicted bounding boxes
"""
grid, boxes = self.dataset[sample_id]
grid, boxes = grid.cpu().squeeze(), boxes.cpu()
frame_depth, frame_width, frame_length = self.dataset.grid_size
detector_out_shape = (1, self.model.out_channels,
frame_width // (2**self.model.n_pools),
frame_length // (2**self.model.n_pools))
gt_former = GroundTruthFormer((frame_width, frame_length),
detector_out_shape,
device=self.device)
fig = plt.figure(figsize=(12, 24))
ax_gt = fig.add_subplot(2, 1, 1)
ax_pred = fig.add_subplot(2, 1, 2)
# plot gt bboxes
ax_gt = draw_bev_with_bboxes(grid, boxes, edgecolor="red", ax=ax_gt)
mapped_bb, mapped_bb_3sigma, mapped_bb_n3sigma = self.get_bbox_from_regression(
mean_class, mean_regr, sigma_regr, gt_former.prior_boxes_params)
ax_pred = draw_bev_with_bboxes(grid,
mapped_bb_3sigma.cpu(),
edgecolor="red",
label="model confidence 98%",
ax=ax_pred)
ax_pred = draw_bev_with_bboxes(grid,
mapped_bb.cpu(),
edgecolor="darkred",
ax=ax_pred,
label="model confidence 50%")
ax_pred = draw_bev_with_bboxes(grid,
mapped_bb_n3sigma.cpu(),
edgecolor="lightcoral",
ax=ax_pred,
label="model confidence 2%")
ax_pred.legend()
return fig, ax_gt, ax_pred
def get_bbox_from_regression(
self, mean_class: torch.Tensor, mean_regr: torch.Tensor,
sigma_regr: torch.Tensor, prior_boxes: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Calculate bboxes (of mean values, mean - sigma, mean + 3 sigma) from Monte Carlo estimations
:param mean_class: mean of classification predictions
:param mean_regr: mean of regression predictions
:param sigma_regr: standard deviation of regression predictions
:param prior_boxes: prior boxes from GroundTruthFormer
:return: tuple of three torch.Tensors:
1st - bbox with confidence 50% (from mean)
2nd - bbox with confidence 98% (from mean + 3 * sigma)
3rd - bbox with confidence 2%% (from mean - 3 * sigma)
"""
prior_boxes = prior_boxes[(torch.sigmoid(mean_class) >
self.threshold).squeeze()]
unmapped_bb = mean_regr.squeeze()[(torch.sigmoid(mean_class.squeeze())
> self.threshold)]
mapped_bb = torch.zeros_like(unmapped_bb)
mapped_bb[:, 2:4] = prior_boxes[:, 2:4] / torch.clamp(
torch.exp(unmapped_bb[:, 2:4]), min=1e-6)
mapped_bb[:, 0:2] = prior_boxes[:, 0:2] - (unmapped_bb[:, 0:2] *
mapped_bb[:, 2:4])
mapped_bb[:, 4] = unmapped_bb[:, 4]
mapped_bb[:, 5] = unmapped_bb[:, 5]
mapped_bb_3sigma = mapped_bb.clone()
# forward propagation of uncertainty for non-linear case:
propagated_std = prior_boxes[:, 2:4] * (-torch.exp(-unmapped_bb[:, 2:4])) * \
sigma_regr.squeeze()[(torch.sigmoid(mean_class.squeeze()) > self.threshold)][:, 2:4]
mapped_bb_3sigma[:, 2:4] -= 3 * propagated_std
mapped_bb_neg_3sigma = mapped_bb.clone()
mapped_bb_neg_3sigma[:, 2:4] += 3 * propagated_std
return mapped_bb, mapped_bb_3sigma, mapped_bb_neg_3sigma
def train(output_model_dir: str, input_model_path: Optional[str] = None, tb_path: str = None,
nuscenes_version: str = 'v1.0-mini', data_path: str = "data/v1.0-mini", n_scenes: int = None,
learning_rate: int = 1e-4, n_dumps_per_epoch: int = 10, n_loader_workers: int = 4, batch_size: int = 12,
n_epochs: int = 50, device_id: List[int] = None) -> None:
"""
Train model, log training statistics if tb_path is specified.
:param output_model_dir: path to directory to save model weights to
:param input_model_path: path to model weights. If None, create new model
:param tb_path: name of the folder for tensorboard data to be store in
:param nuscenes_version: version of the dataset
:param data_path: relative path to data folder
:param n_scenes: number of scenes in dataset
:param learning_rate: learning rate for Adam
:param n_dumps_per_epoch: how many times per epoch to dump images to tensorboard (not implemented yet)
:param n_loader_workers: number of CPU workers for data loader processing
:param batch_size: batch size
:param n_epochs: total number of epochs to train the model
:param device_id: list of gpu device ids to use, e.g [0, 1]
"""
# create path for model save
os.makedirs(output_model_dir, exist_ok=True)
# set up computing device for pytorch
if torch.cuda.is_available():
if device_id is None:
device_id = [0]
if max(device_id) < torch.cuda.device_count():
# device_id/s all exist on machine,
# device is set as a root device
device = torch.device(f'cuda:{device_id[0]}')
else:
# device_id is out of range, setting to defaults cuda:0
print('Warning: specified number of gpu device_id is larger than available, using cuda:0.')
device = torch.device('cuda:0')
print('Using device: GPU\n')
else:
device = torch.device('cpu')
print('Using device: CPU\n')
date = datetime.datetime.now().strftime('%b-%d-%Y-%H:%M:%S')
# set up tensorboard writer
if tb_path is not None:
train_writer = SummaryWriter(log_dir=f'{tb_path}/{date}/train')
val_writer = SummaryWriter(log_dir=f'{tb_path}/{date}/val')
print(f'Logging tensorboard data to directory: {tb_path}/{date}\n')
else:
train_writer, val_writer = None, None
print(f'No tensorboard logging will be performed\n')
# set up dataset and model
nuscenes = create_nuscenes(data_path, nuscenes_version)
train_dataset = NuscenesBEVDataset(nuscenes=nuscenes, n_scenes=n_scenes, mode='train')
val_dataset = NuscenesBEVDataset(nuscenes=nuscenes, n_scenes=n_scenes, mode='val')
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=n_loader_workers,
collate_fn=frames_bboxes_collate_fn, pin_memory=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True, num_workers=n_loader_workers,
collate_fn=frames_bboxes_collate_fn, pin_memory=True)
print('Loaders are ready.',
f'Number of batches in train loader: {len(train_loader)}'
f'Number of batches in validation loader: {len(val_loader)}', sep='\n')
frame_depth, frame_width, frame_length = train_dataset.grid_size
model = Detector(img_depth=frame_depth)
if input_model_path is not None:
model.load_state_dict(torch.load(input_model_path, map_location="cpu"))
model = model.to(device)
criterion = DetectionLoss()
optimizer = Adam(model.parameters(), lr=learning_rate)
scheduler = StepLR(optimizer, gamma=0.5, step_size=50) # TODO: adjust step_size empirically
detector_out_shape = (batch_size, model.out_channels, frame_width // (2 ** model.n_pools),
frame_length // (2 ** model.n_pools))
gt_former = GroundTruthFormer((frame_width, frame_length), detector_out_shape, device=device)
if len(device_id) > 1 and max(device_id) < torch.cuda.device_count():
# if more than one device_id specified, use DataParallel
model = nn.DataParallel(model, device_ids=device_id)
model = model.to(device)
best_val_score = float('-inf')
for epoch in trange(n_epochs, desc="Epoch"):
run_epoch(model, train_loader, criterion, gt_former, epoch, mode='train',
writer=train_writer, optimizer=optimizer, device=device)
scheduler.step()
val_loss, val_score = run_epoch(model, val_loader, criterion, gt_former, epoch,
mode='val', train_loader_size=len(train_loader), writer=val_writer,
device=device)
# saving model weights in case validation loss AND score are better
if val_score > best_val_score:
best_val_score = val_score
torch.save(model.state_dict(), f'{output_model_dir}/{date}.pth')
print('\nModel checkpoint is saved.', f'loss: {val_loss:.3f}, score: {val_score:.3f}', sep='\n')
model = torchvision.models.resnet50(pretrained=True).to(args.device)
model.eval()
def extract(self, input, output):
if output.ndimension() > 2:
features = F.avg_pool2d(output, output.shape[-2:]).squeeze(3).squeeze(2)
features_state[self] = features
blocks = itertools.chain(model.layer1, model.layer2, model.layer3, model.layer4, (model.avgpool,))
for b in blocks:
b.register_forward_hook(extract)
detector = Detector(**vars(args)).to(args.device)
if args.eval:
ckpt_path = os.path.join(args.run_dir, 'ckpt', 'best_model.pth')
if not os.path.exists(ckpt_path):
print('No pretrained model found:', ckpt_path)
sys.exit(1)
print('Loading:', ckpt_path)
ckpt = torch.load(ckpt_path)
detector.load_state_dict(ckpt['detector'])
test_data = OrigAdvDataset(test_data, orig_transform=transform, return_paths=True)
test_cache = 'cache/cache_test_{}_{}.pth'.format('cos' if args.distance == 'cosine' else 'euc', 'med' if 'medoids' in args.centroids else 'centr')
test_paths, test_data = precompute_embeddings(features_state, test_data, model, args, return_paths=True, cache=test_cache)
test_loader = DataLoader(test_data, shuffle=False, pin_memory=True, num_workers=8, batch_size=args.batch_size)
def main(pretrained, data_files, model_params, training_params, device):
""" forward pass dev and eval data to trained model """
batch_size = training_params['batch_size']
test_batch_size = training_params['test_batch_size']
epochs = training_params['epochs']
start_epoch = training_params['start_epoch']
n_warmup_steps = training_params['n_warmup_steps']
log_interval = training_params['log_interval']
kwargs = {
'num_workers': 4,
'pin_memory': True
} if device == torch.device('cuda') else {}
# create model
model = Detector(**model_params).to(device)
num_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print('===> Model total parameter: {}'.format(num_params))
if device == torch.device('cuda') and torch.cuda.device_count() > 1:
print('multi-gpu')
model = nn.DataParallel(model).cuda()
if pretrained:
epoch_id = pretrained.split('/')[2].split('_')[0]
pretrained_id = pretrained.split('/')[1]
if os.path.isfile(pretrained):
print("===> loading checkpoint '{}'".format(pretrained))
checkpoint = torch.load(
pretrained,
map_location=lambda storage, loc: storage) # load for cpu
best_acc1 = checkpoint['best_acc1']
model.load_state_dict(checkpoint['state_dict'])
print("===> loaded checkpoint '{}' (epoch {})".format(
pretrained, checkpoint['epoch']))
else:
print("===> no checkpoint found at '{}'".format(pretrained))
exit()
else:
raise NameError
# Data loading code (class analysis for multi-class classification only)
val_data = SpoofDatsetSystemID(data_files['dev_scp'],
data_files['dev_utt2index'],
binary_class=False)
eval_data = SpoofDatsetSystemID(data_files['eval_scp'],
data_files['eval_utt2index'],
binary_class=False)
val_loader = torch.utils.data.DataLoader(val_data,
batch_size=test_batch_size,
shuffle=False,
**kwargs)
eval_loader = torch.utils.data.DataLoader(eval_data,
batch_size=test_batch_size,
shuffle=False,
**kwargs)
os.makedirs(data_files['scoring_dir'], exist_ok=True)
# forward pass for dev
print("===> forward pass for dev set")
score_file_pth = os.path.join(
data_files['scoring_dir'],
str(pretrained_id) + '-epoch%s-dev_scores.txt' % (epoch_id))
print("===> dev scoring file saved at: '{}'".format(score_file_pth))
prediction.prediction(val_loader, model, device, score_file_pth,
data_files['dev_utt2systemID'])
# forward pass for eval
print("===> forward pass for eval set")
score_file_pth = os.path.join(
data_files['scoring_dir'],
str(pretrained_id) + '-epoch%s-eval_scores.txt' % (epoch_id))
print("===> eval scoring file saved at: '{}'".format(score_file_pth))
prediction.prediction(eval_loader, model, device, score_file_pth,
data_files['eval_utt2systemID'])
def main():
device = torch.device("cuda:0" if args.cuda else "cpu")
print(device)
#model = VAE([128,128], lr=args.lr, eps=args.eps)
model = Detector()
model = resnet18(pretrained=False, progress=False)
#model_path = '/hdd_c/data/miniWorld/trained_models/Detector/dataset_5/Detector.pth'
model_path = '/hdd_c/data/miniWorld/trained_models/Detector/dataset_5/Detector_resnet18.pth'
data_path = '/hdd_c/data/miniWorld/dataset_5/'
all_obs, all_y = read_data(data_path, max_num_eps=400)
# normalize
all_obs = all_obs/255.0
all_obs = np.swapaxes(all_obs,1,3)
y_max_dir = np.amax(all_y[:,:,1])
y_min_dir = np.amin(all_y[:,:,1])
y_max_dis = np.amax(all_y[:,:,2])
y_min_dis = np.amin(all_y[:,:,2])
print(y_max_dir)
print(y_min_dir)
print(y_max_dis)
print(y_min_dis)
all_y[:,:,1] = (all_y[:,:,1]-y_min_dir)/(y_max_dir-y_min_dir)
all_y[:,:,2] = (all_y[:,:,2]-y_min_dis)/(y_max_dis-y_min_dis)
#print(all_y[:10,:,:])
#raise Error
# split
split_point = int(all_obs.shape[0]*0.8)
all_obs_train = all_obs[:split_point]
all_obs_val = all_obs[split_point:]
all_y_train = all_y[:split_point]
all_y_val = all_y[split_point:]
# train with shuffle
indices = np.arange(all_obs_train.shape[0])
np.random.shuffle(indices)
all_obs_train = all_obs_train[indices]
all_y_train = all_y_train[indices]
# val with shuffle
indices = np.arange(all_obs_val.shape[0])
np.random.shuffle(indices)
all_obs_val = all_obs_val[indices]
all_y_val = all_y_val[indices]
print('Available number of obs: {}'.format(len(all_obs)))
print(all_obs_train.shape)
#data_train = all_obs[:200000]
#data_eval = all_obs[200000:240000]
#image = np.zeros([32,3,128,128])
#image = make_var(image)
#z = model.encode(image)
#r = model.decode(z)
#dummy_data = np.ones([6400,3,128,128])
#print(data_eval.shape)
training_instance = trainDetector(device, model, lr=args.lr, eps=args.eps, input_train=all_obs_train, y_train=all_y_train, input_eval=all_obs_val, y_eval=all_y_val, model_path=model_path)
training_instance.train()
pin_memory=False,
drop_last=True)
val_dataset = ImageDataset("valid",
args.image_size,
args.file_loc,
args.shuffle,
fold=0,
do_transform=False)
val_loader = DataLoader(val_dataset,
num_workers=0,
shuffle=True,
batch_size=args.batch_size,
pin_memory=False,
drop_last=True)
model = Detector(args.dropout_rate).cuda()
set_requires_grad([model.feature_extractor], False)
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-6)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer,
T_max=args.epochs,
eta_min=args.min_lr,
last_epoch=-1)
bce_loss = torch.nn.BCEWithLogitsLoss()
epoch = 0
step = 0
writer = SummaryWriter(args.logging_path)
if args.resume_from_last:
args.checkpoint_path = get_last_checkpoint_filename(args.logging_path)