def loss_epoch(net: nn.Module,
dataloader: BaseDataLoader,
loss_func: nn.Module,
metrics: List[nn.Module],
device: torch.device,
optimizer: Optimizer = None,
tqdm_description: str = None,
max_batches: int = None) -> dict:
"""
:param max_batches: max number of batches to process. use to perform sanity check
"""
# create `epoch_stats` dict
epoch_stats = {utils.get_class_name(metric): 0 for metric in metrics}
# generate field to store loss in case it's not present in `metrics` list
epoch_stats[utils.get_class_name(loss_func)] = 0
n_samples = len(dataloader)
gen = dataloader.get_generator()
with tqdm.tqdm(total=n_samples,
desc=tqdm_description,
unit='slice',
leave=True,
bar_format=const.TQDM_BAR_FORMAT) as pbar_t:
for batch_ix, (scans_batch, masks_batch,
descriptions_batch) in enumerate(gen, start=1):
batch_size = len(scans_batch)
batch_stats = loss_batch(net=net,
x_batch=scans_batch,
y_batch=masks_batch,
loss_func=loss_func,
metrics=metrics,
device=device,
optimizer=optimizer)
# TODO: what are the issues of using `reduction='sum'`?
# there was one as I remember
# add values of batch stats to epoch stats.
# multiply batch_stats by batch_size to handle reduction='mean'
epoch_stats = {
k: epoch_stats[k] + batch_stats[k] * batch_size
for k in epoch_stats.keys() & batch_stats.keys()
}
pbar_t.update(batch_size)
if max_batches is not None and batch_ix >= max_batches:
break
# average stats
epoch_stats = {k: v / n_samples for k, v in epoch_stats.items()}
return epoch_stats
def __init__(self,
net: nn.Module,
loss_func: nn.Module,
optimizer: Optimizer,
train_loader: BaseDataLoader,
lr_min: float = 1e-8,
lr_max: float = 5e1,
beta=0.97,
device: torch.device = torch.device('cuda:0')):
"""
:param beta: smoothing factor for exponentially weighted window
"""
self.net = net
self.loss_func = loss_func
self.loss_name = utils.get_class_name(loss_func)
self.optimizer = optimizer
self.train_loader = train_loader
self.n_batches = train_loader.n_batches
self.lr_min = lr_min
self.lr_max = lr_max
self.q = (self.lr_max / self.lr_min)**(1 / (self.n_batches - 1))
self.beta = beta
self.device = device
self.lrs = None
self.losses = None
self.raw_losses = None
def __getitem__(self, index):
if hasattr(self, index):
return Ref(self, index)
else:
class_name = get_class_name(self)[0]
raise ValueError('%s state does not have attribute "%s".' %
(class_name, index))
def __getitem__(self, index):
if hasattr(self, index):
return Ref(self, index)
else:
class_name = get_class_name(self)[0]
raise ValueError('%s state does not have attribute "%s".' %
(class_name, index))
def set_vector_matrix(self, vectors):
assert isinstance(vectors, np.ndarray), \
'vectors must be a {} instance'.format(get_class_name(np.ndarray))
assert vectors.shape == (self.vocab_size, self.vector_size), \
'dimension of vectors {} mismatch with ({}, {})'.format(
vectors.shape, self.vocab_size, self.vector_size)
self.word2vec.syn0 = deepcopy(vectors)
self.word2vec.syn0norm = self.word2vec.syn0
def __getitem__(self, index):
"""
Returns a reference object for the specified attribute.
"""
if hasattr(self, index):
return Ref(self, index)
else:
class_name = get_class_name(self)[0]
raise ValueError('%s state does not have attribute "%s".' %
(class_name, index))
def create(self):
try:
module = importlib.import_module('strategies.{0}'.format(self.strategy))
except ImportError:
logging.exception('import error')
else:
strategy_module = getattr(module, get_class_name(self.strategy))
return strategy_module(
initial_money = self.initial_money,
initial_bet = self.initial_bet
)
def loss_batch(net: nn.Module,
x_batch,
y_batch,
loss_func: nn.Module,
metrics: List[nn.Module],
device: torch.device,
optimizer: Optimizer = None) -> dict:
batch_stats = {}
x = torch.tensor(x_batch, dtype=torch.float, device=device).unsqueeze(1)
y = torch.tensor(y_batch, dtype=torch.float, device=device).unsqueeze(1)
out = net(x)
loss = loss_func(out, y)
loss_name = utils.get_class_name(loss_func)
batch_stats[loss_name] = loss.item()
if optimizer is not None:
optimizer.zero_grad()
loss.backward()
optimizer.step()
# It's uncommon to set model to evaluation state with `net.eval()`
# to calculate additional metrics during training because:
# * it's resource expensive
# * metrics calculated with `net.eval()` are similar to ones with `net.train()`
# According to https://discuss.pytorch.org/t/model-eval-for-train-accuracy/61526
# So we'll just disable gradient tracking.
# `net.eval`() must be called outside this function when calculating metrics on validation set
with torch.no_grad():
for metric_func in metrics:
metric_name = utils.get_class_name(metric_func)
if metric_name != loss_name:
metric_val = metric_func(out, y)
batch_stats[metric_name] = metric_val.item()
return batch_stats
def example(img, top_k=3):
saliency = explainer(img.to(device)).cpu().numpy()
p, c = torch.topk(model(img.to(device)), k=top_k)
p, c = p[0], c[0]
plt.figure(figsize=(10, 5 * top_k))
for k in range(top_k):
plt.subplot(top_k, 2, 2 * k + 1)
plt.axis('off')
plt.title('{:.2f}% {}'.format(100 * p[k], get_class_name(c[k])))
tensor_imshow(img[0])
plt.subplot(top_k, 2, 2 * k + 2)
plt.axis('off')
plt.title(get_class_name(c[k]))
tensor_imshow(img[0])
sal = saliency[c[k]]
plt.imshow(sal, cmap='jet', alpha=0.5)
plt.colorbar(fraction=0.046, pad=0.04)
plt.savefig('0-explain.png')
# plt.show()
plt.close()
def single_run(self, img_tensor, explanation, verbose=0, save_to=None):
"""Run metric on one image-saliency pair.
Args:
img_tensor (Tensor): normalized image tensor.
explanation (np.ndarray): saliency map.
verbose (int): in [0, 1, 2].
0 - return list of scores.
1 - also plot final step.
2 - also plot every step and print 2 top classes.
save_to (str): directory to save every step plots to.
Return:
scores (nd.array): Array containing scores at every step.
"""
pred = self.model(img_tensor.cuda())
top, c = torch.max(pred, 1)
c = c.cpu().numpy()[0]
n_steps = (HW + self.step - 1) // self.step
if self.mode == 'del':
title = 'Deletion game'
ylabel = 'Pixels deleted'
start = img_tensor.clone()
finish = self.substrate_fn(img_tensor)
elif self.mode == 'ins':
title = 'Insertion game'
ylabel = 'Pixels inserted'
start = self.substrate_fn(img_tensor)
finish = img_tensor.clone()
scores = np.empty(n_steps + 1)
# Coordinates of pixels in order of decreasing saliency
t_r = explanation.reshape(-1, HW)
salient_order = np.argsort(t_r, axis=1)
salient_order = torch.flip(salient_order, [0, 1])
for i in range(n_steps + 1):
pred = self.model(start.cuda())
pr, cl = torch.topk(pred, 2)
if verbose == 2:
print('{}: {:.3f}'.format(get_class_name(cl[0][0]),
float(pr[0][0])))
print('{}: {:.3f}'.format(get_class_name(cl[0][1]),
float(pr[0][1])))
scores[i] = pred[0, c]
# Render image if verbose, if it's the last step or if save is required.
if verbose == 2 or (verbose == 1 and i == n_steps) or save_to:
plt.figure(figsize=(10, 5))
plt.subplot(121)
plt.title('{} {:.1f}%, P={:.4f}'.format(
ylabel, 100 * i / n_steps, scores[i]))
plt.axis('off')
tensor_imshow(start[0])
plt.subplot(122)
plt.plot(np.arange(i + 1) / n_steps, scores[:i + 1])
plt.xlim(-0.1, 1.1)
plt.ylim(0, 1.05)
plt.fill_between(np.arange(i + 1) / n_steps,
0,
scores[:i + 1],
alpha=0.4)
plt.title(title)
plt.xlabel(ylabel)
plt.ylabel(get_class_name(c))
if save_to:
plt.savefig(save_to + '/{:03d}.png'.format(i))
plt.close()
if i < n_steps:
coords = salient_order[:, self.step * i:self.step * (i + 1)]
start.cpu().numpy().reshape(
1, 3,
HW)[0, :,
coords] = finish.cpu().numpy().reshape(1, 3,
HW)[0, :,
coords]
return auc(scores)
def main():
############################################################################
# load our training data set of images examples
program_start = cv2.getTickCount()
print("Loading images...")
start = cv2.getTickCount()
# N.B. specify data path names in same order as class names (neg, pos)
paths = [params.DATA_training_path_neg, params.DATA_training_path_pos]
# build a list of class names automatically from our dictionary of class (name,number) pairs
class_names = [utils.get_class_name(class_number) for class_number in range(len(params.DATA_CLASS_NAMES))]
# specify number of sub-window samples to take from each positive and negative
# example image in the data set
# N.B. specify in same order as class names (neg, pos) - again
sampling_sizes = [params.DATA_training_sample_count_neg, params.DATA_training_sample_count_pos]
# do we want to take samples only centric to the example image or ramdonly?
# No - for background -ve images (first class)
# Yes - for object samples +ve images (second class)
sample_from_centre = [False, True];
# perform image loading
imgs_data = utils.load_images(paths, class_names, sampling_sizes, sample_from_centre,
params.DATA_WINDOW_OFFSET_FOR_TRAINING_SAMPLES, params.DATA_WINDOW_SIZE);
print(("Loaded {} image(s)".format(len(imgs_data))))
utils.print_duration(start)
############################################################################
# perform HOG feature extraction
print("Computing HOG descriptors...") # for each training image
start = cv2.getTickCount()
#each HoG descriptor is stored in its respective img_data instance
[img_data.compute_hog_descriptor() for img_data in imgs_data]
utils.print_duration(start)
############################################################################
# train an SVM based on these norm_features
print("Training SVM...")
start = cv2.getTickCount()
# define SVM parameters
svm = cv2.ml.SVM_create()
svm.setType(cv2.ml.SVM_C_SVC) # set SVM type
svm.setKernel(params.HOG_SVM_kernel) # use specific kernel type
# get hog descriptor for each image and store in single global array
samples = utils.get_hog_descriptors(imgs_data)
# get class label for each training image (i.e. 0 for other, 1 for pedestrian... can extend)
class_labels = utils.get_class_labels(imgs_data);
# specify the termination criteria for the SVM training
svm.setTermCriteria((cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_COUNT, params.HOG_SVM_max_training_iterations, 1.e-06))
# perform auto training for the SVM which will essentially perform grid
# search over the set of parameters for the chosen kernel and the penalty
# cost term, C (N.B. trainAuto() syntax is correct as of OpenCV 3.4.x)
svm.trainAuto(samples, cv2.ml.ROW_SAMPLE, class_labels, kFold = 10, balanced = True);
# save the trained SVM to file so that we can load it again for testing / detection
svm.save(params.HOG_SVM_PATH_TRAIN)
############################################################################
# measure performance of the SVM trained on the bag of visual word features
# perform prediction over the set of examples we trained over
output = svm.predict(samples)[1].ravel()
error = (np.absolute(class_labels.ravel() - output).sum()) / float(output.shape[0])
# we are succesful if our prediction > than random
# e.g. for 2 class labels this would be 1/2 = 0.5 (i.e. 50%)
if error < (1.0 / len(params.DATA_CLASS_NAMES)):
print("Trained SVM obtained {}% training set error".format(round(error * 100,2)))
print("-- meaining the SVM got {}% of the training examples correct!".format(round((1.0 - error) * 100,2)))
else:
print("Failed to train SVM. {}% error".format(round(error * 100,2)))
utils.print_duration(start)
print(("Finished training HoG detector. {}".format(format_time(get_elapsed_time(program_start)))))
# Save explanations if needed.
# explanations.tofile('exp_{:05}-{:05}.npy'.format(args.range[0], args.range[-1]))
for i, (img, _) in enumerate(data_loader):
p, c = torch.max(model(img.to(device)), dim=-1)
p, c = p[0].item(), c[0].item()
prob = torch.softmax(model(img.to(device)), dim=-1)
pred_prob = prob[0][c]
plt.figure(figsize=(10, 5))
plt.suptitle('RISE Explanation for model {}'.format(args.model))
plt.subplot(121)
plt.axis('off')
plt.title('{:.2f}% {}'.format(100 * p, get_class_name(c)))
tensor_imshow(img[0])
plt.subplot(122)
plt.axis('off')
plt.title(get_class_name(c))
tensor_imshow(img[0])
sal = explanations[i]
plt.imshow(sal, cmap='jet', alpha=0.5)
# plt.colorbar(fraction=0.046, pad=0.04)
plt.savefig('{}-explain-{}.png'.format(i + 1, args.model))
# plt.show()
plt.close()
def train_valid(net: nn.Module,
loss_func: nn.Module,
metrics: List[nn.Module],
train_loader: BaseDataLoader,
valid_loader: BaseDataLoader,
optimizer: Optimizer,
scheduler: ReduceLROnPlateau,
device: torch.device,
n_epochs: int,
es_tolerance: float,
es_patience: int,
out_dp: str,
max_batches: int = None) -> dict:
"""
:param out_dp: path to dir where to store checkpoints, history and learning curves plot
:param max_batches: max number of batches to process on each epoch. use to perform sanity check
:param es_tolerance: early stopping tolerance
:param es_patience: number of epochs with no significant improvements for early stopping to fire
Returns
-------
dict with training and validation history of following structure:
'<metric name>' : {'train': List[float], 'valid': Lists[float]}
"""
print(const.SEPARATOR)
print('train_valid():')
checkpoints_dp = os.path.join(out_dp, const.MODEL_CHECKPOINTS_DN)
os.makedirs(checkpoints_dp, exist_ok=True)
history = {
utils.get_class_name(m): {
'train': [],
'valid': []
}
for m in metrics
}
loss_name = utils.get_class_name(loss_func)
es_cnt = 0 # early stopping counter
best_loss_valid = float('inf')
best_epoch_ix = -1 # index of best epoch starting from 1
best_net_params = copy.deepcopy(net.state_dict())
print(f'\ntrain parameters:\n\n'
f'model architecture: {utils.get_class_name(net)}\n'
f'loss function: {loss_name}\n'
f'optimizer: {optimizer}\n'
f'number of epochs: {n_epochs}\n'
f'es tolerance: {es_tolerance : .3e}\n'
f'es patience: {es_patience}\n'
f'device: {device}\n'
f'checkpoints dir: {os.path.abspath(checkpoints_dp)}\n'
f'out dp: "{out_dp}"\n'
f'max_batches: {max_batches}')
print(f'\ntrain_loader:\n{train_loader}')
print(f'\nvalid_loader:\n{valid_loader}')
# count global time of training
time_start_train_valid = time.time()
for cur_epoch in range(1, n_epochs + 1):
print(f'\n{"=" * 15} epoch {cur_epoch}/{n_epochs} {"=" * 15}')
time_start_epoch = time.time()
lr_on_epoch_start = get_lr_from_optimizer_first_group(optimizer)
print(f'current learning rate: {lr_on_epoch_start : .3e}\n')
# ----------- training ----------- #
net.train()
tqdm_description = f'epoch {cur_epoch}. training'
epoch_stats_train = loss_epoch(net=net,
dataloader=train_loader,
loss_func=loss_func,
metrics=metrics,
device=device,
optimizer=optimizer,
tqdm_description=tqdm_description,
max_batches=max_batches)
for m_name, val in epoch_stats_train.items():
tqdm.tqdm.write(f'training.\t{m_name}:\t{val : .4f}')
tqdm.tqdm.write(f'time elapsed for training: '
f'{utils.get_elapsed_time_str(time_start_epoch)}')
# ----------- validation ----------- #
time_start_epoch_valid = time.time()
net.eval()
tqdm_description = f'epoch {cur_epoch}. validation'
tqdm.tqdm.write('')
with torch.no_grad():
epoch_stats_valid = loss_epoch(
net=net,
dataloader=valid_loader,
loss_func=loss_func,
metrics=metrics,
device=device,
optimizer=None, # provide no optimizer to avoid backpropagation
tqdm_description=tqdm_description,
max_batches=max_batches)
for m_name, val in epoch_stats_valid.items():
tqdm.tqdm.write(f'validation.\t{m_name}:\t{val : .4f}')
tqdm.tqdm.write(
f'time elapsed for validation: '
f'{utils.get_elapsed_time_str(time_start_epoch_valid)}')
# ----------- end of epoch ----------- #
# store parameters
torch.save(
net.state_dict(),
os.path.join(checkpoints_dp,
f'cp_{loss_name}_epoch_{cur_epoch:02}.pth'))
# append epoch stats to history
for k in epoch_stats_train.keys() & epoch_stats_valid.keys():
history[k]['train'].append(epoch_stats_train[k])
history[k]['valid'].append(epoch_stats_valid[k])
last_val_loss = history[loss_name]['valid'][-1]
# build learning curves (overwrite existing file on each epoch)
utils.build_learning_curves(metrics=history,
loss_name=loss_name,
out_dp=out_dp)
print(f'\nepoch validation loss: {last_val_loss : .4f}')
# scheduler step
scheduler.step(last_val_loss)
cur_lr = get_lr_from_optimizer_first_group(optimizer)
if cur_lr != lr_on_epoch_start:
# load previous best parameters and continue training
print(
f'LR was reduced. Loading model state dict from previous best epoch:\n'
f'best_epoch_ix: {best_epoch_ix}\n'
f'best_loss_valid: {best_loss_valid : .4f}')
net.load_state_dict(best_net_params)
# print elapsed time for epoch
print(f'\ntime elapsed for epoch: '
f'{utils.get_elapsed_time_str(time_start_epoch)}')
# ----------- early stopping ----------- #
if history[loss_name]['valid'][-1] < best_loss_valid - es_tolerance:
best_loss_valid = history[loss_name]['valid'][-1]
tqdm.tqdm.write(
f'\nepoch {cur_epoch}: new best loss valid: {best_loss_valid : .4f}'
)
best_net_params = copy.deepcopy(net.state_dict())
best_epoch_ix = cur_epoch
es_cnt = 0
else:
es_cnt += 1
if es_cnt >= es_patience:
tqdm.tqdm.write(const.SEPARATOR)
tqdm.tqdm.write(
f'\nEarly Stopping!'
f'\nNo improvements for {es_patience} epochs for {loss_name} metric'
)
break
# ----------- end of training ----------- #
# modify history dict
history = {
'metrics': history,
'loss_name': loss_name,
'best_loss_valid': best_loss_valid,
'best_epoch_ix': best_epoch_ix
}
# save best weights once again
torch.save(best_net_params,
os.path.join(checkpoints_dp, f'cp_{loss_name}_best.pth'))
# load best model
# TODO: check if weights of net outside this function are updated
net.load_state_dict(best_net_params)
# print summary
print(const.SEPARATOR)
print('end of training.')
tqdm.tqdm.write(f'total time elapsed: '
f'{utils.get_elapsed_time_str(time_start_train_valid)}')
print(f'best loss valid: {best_loss_valid : .4f}')
print(f'best epoch: {best_epoch_ix}')
return history
def _add_cluster(self, graph, cluster):
# create the cluster
name,cl_uname = get_class_name(cluster)
clust = pydot.Cluster(cl_uname, label=name)
# loop over children of cluster
nodes = []
for i,c in enumerate(cluster.children):
if issubclass(c.__class__, ParentState):
# call recursively
uname,first_uname,last_uname = self._add_cluster(clust, c)
# save in node list
if not uname is None:
nodes.append({'uname':uname,
'first_uname': first_uname,
'last_uname': last_uname})
else:
# add the child node
name,uname = get_class_name(c)
clust.add_node(pydot.Node(uname, label=name))
# save in node list (no children for first/last)
nodes.append({'uname':uname,
'first_uname': None,
'last_uname': None})
# add edges if necessary
if issubclass(cluster.__class__, Serial) and i>0:
# set defaults
ledge = nodes[i-1]['uname']
redge = nodes[i]['uname']
ltail = None
lhead = None
if nodes[i-1]['last_uname']:
# coming from cluster
ledge = nodes[i-1]['last_uname']
ltail = nodes[i-1]['uname'].get_name()
if nodes[i]['first_uname']:
# going to cluster
redge = nodes[i]['first_uname']
lhead = nodes[i]['uname'].get_name()
if ltail and lhead:
self.edges.append(pydot.Edge(ledge, redge,
ltail=ltail,
lhead=lhead))
elif ltail:
self.edges.append(pydot.Edge(ledge, redge,
ltail=ltail))
elif lhead:
self.edges.append(pydot.Edge(ledge, redge,
lhead=lhead))
else:
self.edges.append(pydot.Edge(ledge, redge))
if len(nodes) > 0:
# insert the cluster to the graph
graph.add_subgraph(clust)
if nodes[0]['first_uname']:
first_uname = nodes[0]['first_uname']
else:
first_uname = nodes[0]['uname']
if nodes[-1]['last_uname']:
last_uname = nodes[-1]['last_uname']
else:
last_uname = nodes[-1]['uname']
else:
clust, first_uname, last_uname = None,None,None
# return the cluster uname for connections
return clust, first_uname, last_uname
"classifier": [200, 0.5, 150, 0.5]
}
#################################################
weights_path = '/Users/idofarhi/Documents/Thesis/fold6model.pt'
model = create_model(hyper_params)
load_model(model, weights_path)
model.eval()
sum_predictions = None
for pickle_clip in pickle_clips_path.iterdir():
if str(pickle_clip)[-9:] == '.DS_Store': continue
prep_clip = prep_pickle_for_inference(pickle_clip)
preds = model(prep_clip).squeeze(0) # squeeze to remove batch dimension
softmax_preds = torch.nn.functional.softmax(preds, dim=0)
# print(softmax_preds)
if sum_predictions is None:
sum_predictions = softmax_preds
else:
sum_predictions += softmax_preds
print('The final prediction vector is:', sum_predictions.tolist())
predicted_class = get_class_name(sum_predictions.max(0)[1].item())
print('The final predicted class is:', predicted_class)
min_depth = 100 # initialize to then store what the closest detection is
min_depth_class = "No Detections" # by default in case there are no detections
units = " meters"
# for each detection on the image
for i in range(len(detection_classes)):
# get rect
det_rect = detection_rects[i]
# extract vertex data
x1, y1, x2, y2 = det_rect
# different route depending on model
if model == "SVM":
# get class number
det_class = int(detection_classes[i])
# get class name based on class number
det_class_name = utils.get_class_name(det_class)
# get color based on class number
color = Colors[det_class]
elif model == "MRCNN":
# get class name
det_class_name = detection_class_names[i]
# get color based on class number
color = Colors[detection_classes[i]]
# get confidence
confidence = str(round(confidences[i], 2))
# get depth
det_depth = round(detection_depths[i], 1)
# draw colored rectangle where detected object is
cv2.rectangle(imgL, (x1, y1), (x2, y2), color, 2)