def get_p_cond(distances: tensor, sigmas_sq: tensor, mask: tensor) -> tensor:
"""
Calculates conditional probability distribution given distances and squared sigmas
:param distances: Matrix of squared distances ||x_i - x_j||^2
:param sigmas_sq: Row vector of squared sigma for each row in distances
:param mask: A mask tensor to set diagonal elements to zero
:return: Conditional probability matrix
"""
logits = -distances / (2 * torch_max(sigmas_sq, EPS).view(-1, 1))
logits.exp_()
masked_exp_logits = logits * mask
normalization = torch_max(masked_exp_logits.sum(1), EPS).unsqueeze(1)
return masked_exp_logits / normalization + 1e-10
def evaluate(model,
data_loader,
device,
acc_history=[],
loss_history=[],
mode="test",
eval_type="both"):
model.eval()
total_samples = len(data_loader.sampler)
num_batch = len(data_loader)
correct_samples = 0
total_loss = 0
criterion = CrossEntropyLoss()
with no_grad():
for data, target in data_loader:
data = data.to(device)
target = target.to(device)
output = log_softmax(model(data), dim=1)
loss = criterion(output, target)
_, pred = torch_max(output, dim=1)
total_loss += loss.item()
correct_samples += pred.eq(target).sum()
avg_loss = total_loss / num_batch
accuracy = 100.0 * correct_samples / total_samples
if mode == "val" or (eval_type == "test" and mode == "test"):
loss_history.append(avg_loss)
acc_history.append(accuracy)
sys.stdout.write(' %s: %.4f -- %s: %.2f \n' %
(mode + "_loss", avg_loss, mode + "_acc", accuracy))
return avg_loss, accuracy
def validation_step(self, validation_loader: DataLoader, model: Module,
loss_function: _Loss, device: torch_device) -> None:
"""Run a single evaluation step.
Args:
validation_loader: the data loader for the validation data.
model: the neural network to be trained.
loss_function: the loss function used for training.
device: the torch device used for the model.
"""
model.eval()
for images, labels in validation_loader:
# Move the data to the device if needed.
if device:
images = images.to(device)
labels = labels.to(device)
# Predict image.
prediction = model(images)
loss = loss_function(prediction, labels)
_, predicted = torch_max(prediction.data, 1)
batch_image_count = labels.size(0)
batch_correct_count = (predicted == labels).sum().item()
self._call_hook(Signals.VALIDATION_EPOCH_BATCH_END,
batch_image_count, batch_correct_count,
loss.item() * batch_image_count)
def train_epoch(model, optimizer, train_loader, loss_history, acc_history,
device, epoch, n_epochs):
total_samples = len(train_loader.dataset)
num_batch = len(train_loader)
model.train()
sum_losses = 0
total_correct_samples = 0
criterion = CrossEntropyLoss()
for i, (data, target) in enumerate(train_loader):
data = data.to(device)
target = target.to(device)
optimizer.zero_grad()
output = log_softmax(model(data), dim=1)
loss = criterion(output, target)
_, pred = torch_max(output, dim=1)
correct_samples = pred.eq(target).sum()
accuracy = 100.0 * correct_samples / len(data)
total_correct_samples += correct_samples
sum_losses += loss.item()
loss.backward()
optimizer.step()
sys.stdout.write(
'\rEpoch %03d/%03d [%03d/%03d] -- %s: %.4f -- %s: %.2f --' %
(epoch, n_epochs, i + 1, num_batch, "train_loss", loss.item(),
"train_acc", accuracy))
loss_history.append(sum_losses / num_batch)
acc_history.append(100.0 * total_correct_samples / total_samples)
def test_evaluation(validation_data, model, criterion):
"""Test trained network
Args:
validation_data (DataLoader): Validation set to perform the evaluation
model (nn.Module): Trained model to be evaluated
criterion (nn.CrossEntropyLoss): criterion to compute loss
Returns:
nn.Module, float, float, float: model, test epoch loss, test error, and test accuracy
"""
total_loss = 0
predicted_ok = 0
total_images = 0
model.eval()
for images, labels in validation_data:
images = images.to(DEVICE)
labels = labels.to(DEVICE)
pred = model(images)
loss = criterion(pred, labels)
total_loss += loss.item() * images.size(0)
_, predicted = torch_max(pred.data, 1)
total_images += labels.size(0)
predicted_ok += (predicted == labels).sum().item()
accuracy = predicted_ok / total_images * 100
error = (1 - predicted_ok / total_images) * 100
epoch_loss = total_loss / len(validation_data.dataset)
return model, epoch_loss, error, accuracy
def predict_image(path_of_image, groupStage):
path_of_model = os_path.join("./CUSTOMIZE_4_USER/MODEL_TRAINING",
groupStage, groupStage + ".pth")
path_of_feature = os_path.join("./CUSTOMIZE_4_USER/MODEL_TRAINING",
groupStage, groupStage + ".npz")
start_time = time()
model = NeuralNet(input_size, hidden_size, num_classes).to(device)
model.load_state_dict(load(path_of_model))
data = np_load(path_of_feature)
[h_max, s_max, v_max] = data['data_max']
[h_min, s_min, v_min] = data['data_min']
img = imread(path_of_image)
img = resize(img, (6000, 4000))
img = img[500:-500, 750:-750, :]
img = cvtColor(img, COLOR_BGR2HSV)
hchan, schan, vchan = split(img)
h_hist = calcHist([img], [0], None, [256], [0, 256]).reshape(256, )
s_hist = calcHist([img], [1], None, [256], [0, 256]).reshape(256, )
v_hist = calcHist([img], [2], None, [256], [0, 256]).reshape(256, )
hMean = np_mean(hchan) / 255
DPV_h_max = np_sum(np_absolute(h_hist - h_max)) / (HEIGHT * WIDTH)
DPV_h_min = np_sum(np_absolute(h_hist - h_min)) / (HEIGHT * WIDTH)
sMean = np_mean(schan) / 255
DPV_s_max = np_sum(np_absolute(s_hist - s_max)) / (HEIGHT * WIDTH)
DPV_s_min = np_sum(np_absolute(s_hist - s_min)) / (HEIGHT * WIDTH)
vMean = np_mean(vchan) / 255
DPV_v_max = np_sum(np_absolute(v_hist - v_max)) / (HEIGHT * WIDTH)
DPV_v_min = np_sum(np_absolute(v_hist - v_min)) / (HEIGHT * WIDTH)
correlation = np_corrcoef(h_hist, s_hist)[0][1]
#image_feature = np_array((hMean, DPV_h_max, DPV_h_min, sMean, DPV_s_max, DPV_s_min, vMean, DPV_v_max, DPV_v_min))
image_feature = np_array((hMean, DPV_h_max, DPV_h_min, sMean, DPV_s_max,
DPV_s_min, correlation))
image_feature = from_numpy(image_feature).to(device).float().view(
1, input_size)
with no_grad():
out_predict = model(image_feature)
_, predicted_result = torch_max(out_predict.data, 1)
original = Tensor([[1, 33, 66, 99]])
# Round xx.xx %
percentage_result = np_round(
mm(out_predict.view(1, num_classes), original.view(num_classes,
1)).item(), 2)
# Processed time
processedTime = np_round(time() - start_time, 2)
#print("Time ",processedTime)
return percentage_result, processedTime
def make_joint(distr_cond: tensor) -> tensor:
"""
Makes a joint probability distribution out of conditional distribution
:param distr_cond: Conditional distribution matrix
:return: Joint distribution matrix. All values in it sum up to 1.
Too small values are set to fixed epsilon
"""
n_points = distr_cond.size(0)
distr_joint = (distr_cond + distr_cond.t()) / (2 * n_points)
return torch_max(distr_joint, EPS)
def random_crop(image, boxes, labels):
original_h = image.size(1)
original_w = image.size(2)
while True:
min_overlap = rand_choice([0., .1, .3, .5, .7, .9, None])
if min_overlap is None:
return image, boxes, labels
max_trials = 50
for _ in range(max_trials):
min_scale = 0.3
scale_h = rand_uniform(min_scale, 1)
scale_w = rand_uniform(min_scale, 1)
new_h = int(scale_h * original_h)
new_w = int(scale_w * original_w)
aspect_ratio = new_h / new_w
if not 0.5 < aspect_ratio < 2:
continue
left = randint(0, original_w - new_w)
right = left + new_w
top = randint(0, original_h - new_h)
bottom = top + new_h
crop = FloatTensor([left, top, right, bottom])
overlap = find_jaccard_overlap(crop.unsqueeze(0), boxes)
overlap = overlap.squeeze(0)
if overlap.max().item() < min_overlap:
continue
new_image = image[:, top:bottom, left:right]
bb_centers = (boxes[:, :2] + boxes[:, 2:]) / 2.
centers_in_crop = (bb_centers[:, 0] > left) * (
bb_centers[:, 0] < right) * (bb_centers[:, 1] >
top) * (bb_centers[:, 1] < bottom)
if not centers_in_crop.any():
continue
new_boxes = boxes[centers_in_crop, :]
new_labels = labels[centers_in_crop]
new_boxes[:, :2] = torch_max(new_boxes[:, :2], crop[:2])
new_boxes[:, :2] -= crop[:2]
new_boxes[:, 2:] = torch_min(new_boxes[:, 2:], crop[2:])
new_boxes[:, 2:] -= crop[:2]
return new_image, new_boxes, new_labels
def _bbox_ious(box1, box2, is_corner_coordinates=True):
"""Calculation of intersection over union function **for many predictions and ground truths** as used in YOLO
rewrite in PyTorch.
This is implemented as shown in https://github.com/CharlesPikachu/YOLO. Modifications are made for variable names
only. All credits to @CharlesPikachu.
"""
x_left = torch_min(box1[0],
box2[0]) if is_corner_coordinates else torch_min(
box1[0] - box1[2] / 2.0, box2[0] - box2[2] / 2.0)
x_right = torch_max(box1[2],
box2[2]) if is_corner_coordinates else torch_max(
box1[0] + box1[2] / 2.0, box2[0] + box2[2] / 2.0)
y_top = torch_min(box1[1],
box2[1]) if is_corner_coordinates else torch_min(
box1[1] - box1[3] / 2.0, box2[1] - box2[3] / 2.0)
y_bottom = torch_max(box1[3],
box2[3]) if is_corner_coordinates else torch_max(
box1[1] + box1[3] / 2.0, box2[1] + box2[3] / 2.0)
box1_width = box1[2] - box1[0] if is_corner_coordinates else box1[2]
box1_height = box1[3] - box1[1] if is_corner_coordinates else box1[3]
box2_width = box2[2] - box2[0] if is_corner_coordinates else box2[2]
box2_height = box2[3] - box2[1] if is_corner_coordinates else box2[3]
raw_union_width = x_right - x_left
raw_union_height = y_bottom - y_top
intersection_width = box1_width + box2_width - raw_union_width
intersection_height = box1_height + box2_height - raw_union_height
mask = ((intersection_width <= 0) + (intersection_height <= 0) > 0)
box1_area = box1_width * box1_height
box2_area = box2_width * box2_height
intersection = intersection_width * intersection_height
intersection[mask] = 0
union = box1_area + box2_area - intersection
return intersection / union
def train(epoch, optimizer_):
loss_list = []
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
data, target = Variable(data), Variable(target)
optimizer_.zero_grad()
output = model(data)
loss = binary_cross_entropy(output, target)
loss_list.append(loss.item())
loss.backward()
optimizer_.step()
if batch_idx % 10 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.item()))
model.eval()
total_correct = 0
total = 0
for batch_idx, (data, target) in enumerate(validation_loader):
data, target = Variable(data), Variable(target)
_, target_inds = torch_max(target, dim=1)
output = model(data)
_, output_inds = torch_max(output, dim=1)
num_correct = torch_eq(target_inds, output_inds).sum().item()
total_correct += num_correct
total += len(target_inds)
accuracy = total_correct / total * 100
print("Model Accuracy: {:.2f}%".format(accuracy))
return loss_list, accuracy
def test_vgg_classifier(self,
epoch: int,
global_epoch: int,
config: LayerTrainingDefinition,
plot_every_n_epochs=1):
r"""
The vgg network has its own classification layer with a cost function that differs from the usual
autoencoder network block. Therefore it has its own test method.
"""
# TODO: figure out if necessaryself self.model.eval()
test_loss = 0
test_acc = 0
# switch to eval mode (dropout)
config.model.eval()
with torch.no_grad():
img: Tensor = None
labels: Tensor = None
for data in self.test_loader:
img, label = data[0], data[1]
# copy all vars to device and calculate the topmost stack representation
# TODO: avoid calculating this representation twice (here and in forward())
dev_img = img.to(self.device)
dev_label = label.to(self.device)
# ===================Forward=====================
prediction = config.model(dev_img)
loss_pred = self.pred_criterion(prediction, dev_label)
# Calculate Test Loss
test_loss += loss_pred.item() / (len(self.test_loader))
# Calculate Accuracy
_, predicted = torch_max(prediction.data, 1)
test_acc += (predicted.cpu().numpy() == label.numpy()
).sum() / (len(self.test_loader) * len(label))
self.test_losses.append(test_loss)
self.test_accs.append(test_acc)
logging.info('Epoch [{}/{}] Test Loss:{:.4f} Test Acc:{:.4f}'.format(
epoch + 1, config.num_epochs, test_loss, test_acc))
self.writer.add_scalar('loss_linear/test',
test_loss,
global_step=global_epoch)
self.writer.add_scalar('accuracy_linear/test',
test_acc,
global_step=global_epoch)
def _find_intersection(set1, set2):
"""Calculation of intersection of every box combination between two sets of boxes that are in boundary
coordinates as used in SSD rewrite in PyTorch.
This is implemented as shown in https://github.com/sgrvinod/a-PyTorch-Tutorial-to-Object-Detection. Some
modifications are made. All credits to @sgrvinod.
"""
lower_bounds = torch_max(set1[:, :2].unsqueeze(1),
set2[:, :2].unsqueeze(0))
upper_bounds = torch_min(set1[:, 2:].unsqueeze(1), set2[:,
2:].unsqueeze(0))
intersection_dims = torch_clamp(upper_bounds - lower_bounds, min=0)
return intersection_dims[:, :, 0] * intersection_dims[:, :, 1]
def accuracy_torch(output: tensor, label: tensor) -> float:
"""Computes accuracy of a model's output.
Parameters
----------
output : tensor
Probabilities predicted by a model.
label : tensor
True labels.
Returns
-------
float
Accuracy of predictions.
"""
_, predicted = torch_max(output.data, 1)
return (predicted == label).sum().item() / len(label)
def get_q_joint(
emb_points: tensor,
dist_func: str,
alpha: int,
) -> tensor:
"""
Calculates the joint probability matrix in embedding space.
:param emb_points: Points in embeddings space
:param alpha: Number of degrees of freedom in t-distribution
:param dist_func: A kay name for a distance function
:return: Joint distribution matrix in emb. space
"""
n_points = emb_points.size(0)
mask = (-eye(n_points) + 1).to(emb_points.device)
dist_f = distance_functions[dist_func]
distances = dist_f(emb_points) / alpha
q_joint = (1 + distances).pow(-(1 + alpha) / 2) * mask
q_joint /= q_joint.sum()
return torch_max(q_joint, EPS)
def get_answer(sentence):
with open(
os.path.join(os.path.dirname(__file__), '..', 'data',
'intents.json'), 'r') as f:
intents = json.load(f)
FILE = os.path.join(os.path.dirname(__file__), '..', 'data', 'data.pth')
data = torch_load(FILE)
input_size = data['input_size']
hidden_size = data['hidden_size']
output_size = data['output_size']
all_words = data['all_words']
tags = data['tags']
model_state = data['model_state']
device = torch_device('cuda' if torch_cuda.is_available() else 'cpu')
model = NeuralNet(input_size, hidden_size, output_size).to(device)
model.load_state_dict(model_state)
model.eval()
sentence = tokenize(sentence)
X = bag_of_words(sentence, all_words)
X = X.reshape(1, X.shape[0])
X = torch_from_numpy(X)
output = model(X)
_, predicted = torch_max(output, dim=1)
tag = tags[predicted.item()]
answer = ''
probs = torch_softmax(output, dim=1)
prob = probs[0][predicted.item()]
if prob.item() > 0.75:
for intent in intents['intents']:
if tag == intent['tag']:
answer = random.choice(intent['responses'])
else:
answer = "I don't understand..."
return answer, prob
def inference_phase(model, criterion, validation_data):
"""Inference phase.
Args:
model (nn.Module): Trained model to be evaluated
criterion (nn.CrossEntropyLoss): criterion to compute loss
validation_data (DataLoader): Validation set to perform the evaluation
"""
# pylint: disable=too-many-locals
total_loss = 0
predicted_ok = 0
total_images = 0
accuracy_pre = 0
error_pre = 0
model.eval()
print('\n ********************************************************* \n')
print(f'\n{datetime.now().time().replace(microsecond=0)} --- '
f'Started LeNet5 Inference')
# Accuracy after the training phase is completed.
for images, labels in validation_data:
images = images.to(DEVICE)
labels = labels.to(DEVICE)
pred = model(images)
loss = criterion(pred, labels)
total_loss += loss.item() * images.size(0)
_, predicted = torch_max(pred.data, 1)
total_images += labels.size(0)
predicted_ok += (predicted == labels).sum().item()
accuracy_pre = predicted_ok/total_images*100
error_pre = (1-predicted_ok/total_images)*100
print(f'{datetime.now().time().replace(microsecond=0)} --- '
f'Error after training: {error_pre:.2f}%\t'
f'Accuracy after training: {accuracy_pre:.2f}%\t')
# Simulation of inference pass at different times after training.
for t_inference in [0., 1., 20., 1000., 1e5]:
model.drift_analog_weights(t_inference)
time_since = t_inference
accuracy_post = 0
error_post = 0
predicted_ok = 0
total_images = 0
for images, labels in validation_data:
images = images.to(DEVICE)
labels = labels.to(DEVICE)
pred = model(images)
loss = criterion(pred, labels)
total_loss += loss.item() * images.size(0)
_, predicted = torch_max(pred.data, 1)
total_images += labels.size(0)
predicted_ok += (predicted == labels).sum().item()
accuracy_post = predicted_ok/total_images*100
error_post = (1-predicted_ok/total_images)*100
print(f'{datetime.now().time().replace(microsecond=0)} --- '
f'Error after inference: {error_post:.2f}%\t'
f'Accuracy after inference: {accuracy_post:.2f}%\t'
f'Drift t={time_since: .2e}\t')
plt.legend(legend, loc='lower right')
plt.show()
# Show a test image
single_image_dataset = SingleImageDataset()
while True:
command = input("Test model on random image (y/n)? ")
if command != 'y':
break
single_image_dataset.load_image(rotor_dataset.random_image_filepath())
data_loader = DataLoader(single_image_dataset, batch_size=1)
data_iter = iter(data_loader)
image = data_iter.next()
# Generate the steering value prediction
output = model(image)
_, result_value = torch_max(output, dim=1)
result_name = Label.label_index_to_name(result_value)
show_image(image, 0, result_value, None)
# Save off the model
save_model = input("Save model (y/n)? ")
if save_model == 'y':
model_export_filepath = str(Path(os.getcwd()) / Path('nn_model.pt'))
torch_save(model.state_dict(), model_export_filepath)
def detect_objects(self, image_as_tensor, confidence_threshold,
nms_threshold):
self.forward(image_as_tensor)
yolov2_loss = self.loss_function.layer
num_anchors = yolov2_loss.num_anchors
if self.predictions.dim() == 3:
self.predictions = self.predictions.unsqueeze(0)
assert self.predictions.size(1) == (5 + self.class_count) * num_anchors
batch_size = self.predictions.size(0)
feature_map_height = self.predictions.size(2)
feature_map_width = self.predictions.size(3)
number_of_pixels = feature_map_height * feature_map_width
view_size = batch_size * num_anchors * number_of_pixels
output = self.predictions.view(
batch_size * num_anchors, 5 + self.class_count,
number_of_pixels).transpose(0, 1).contiguous().view(
5 + self.class_count, view_size)
output = output.cuda() if self.use_cuda else output
grid_x = torch_linspace(0, feature_map_width - 1,
feature_map_width).repeat(
feature_map_height,
1).repeat(batch_size * num_anchors, 1,
1).view(view_size)
grid_x = grid_x.cuda() if self.use_cuda else grid_x
grid_y = torch_linspace(0, feature_map_height - 1,
feature_map_height).repeat(
feature_map_width,
1).t().repeat(batch_size * num_anchors, 1,
1).view(view_size)
grid_y = grid_y.cuda() if self.use_cuda else grid_y
anchor_w = Tensor(yolov2_loss.anchors).view(
num_anchors,
yolov2_loss.anchor_step).index_select(1, LongTensor([0])).repeat(
batch_size, 1).repeat(1, 1, number_of_pixels).view(view_size)
anchor_w = anchor_w.cuda() if self.use_cuda else anchor_w
anchor_h = Tensor(yolov2_loss.anchors).view(
num_anchors,
yolov2_loss.anchor_step).index_select(1, LongTensor([1])).repeat(
batch_size, 1).repeat(1, 1, number_of_pixels).view(view_size)
anchor_h = anchor_h.cuda() if self.use_cuda else anchor_h
class_scores = Softmax()(output[5:5 + self.class_count].transpose(
0, 1)).data
max_class_scores, top_classes = torch_max(class_scores, 1)
objectness_confidences = torch_sigmoid(output[4])
max_class_scores = max_class_scores.view(-1)
top_classes = top_classes.view(-1)
confidences = objectness_confidences * max_class_scores
objectness_confidences = objectness_confidences[
confidences > confidence_threshold]
x_predictions = (torch_sigmoid(output[0]) + grid_x)[
confidences > confidence_threshold] / feature_map_width
y_predictions = (torch_sigmoid(output[1]) + grid_y)[
confidences > confidence_threshold] / feature_map_height
w_predictions = (torch_exp(output[2]) * anchor_w)[
confidences > confidence_threshold] / feature_map_width
h_predictions = (torch_exp(output[3]) * anchor_h)[
confidences > confidence_threshold] / feature_map_height
class_scores = class_scores[confidences > confidence_threshold].view(
-1, self.class_count)
max_class_scores = max_class_scores[confidences > confidence_threshold]
top_classes = top_classes[confidences > confidence_threshold]
all_boxes = []
for b_index in range(batch_size):
boxes = []
for index in range(x_predictions.size(0)):
objectness = objectness_confidences[index].item()
cx = x_predictions[index].item()
cy = y_predictions[index].item()
w = w_predictions[index].item()
h = h_predictions[index].item()
max_class_score = max_class_scores[index].item()
top_class = top_classes[index].item()
box = [cx, cy, w, h, objectness, max_class_score, top_class]
possible_classes = (class_scores[index] * objectness >
confidence_threshold).nonzero()[:, 0]
possible_classes = possible_classes[
possible_classes != top_class]
for cls in possible_classes:
box.append(class_scores[index][cls].item())
box.append(cls.item())
boxes.append(box)
all_boxes.append(boxes)
detections = []
image_width, image_height = image_as_tensor.size(
3), image_as_tensor.size(2)
for b_index in range(batch_size):
batch_detections = []
boxes = nms(all_boxes[b_index], nms_threshold)
for box in boxes:
x1 = max(box[0] - box[2] / 2.0, 0) * image_width
y1 = max(box[1] - box[3] / 2.0, 0) * image_height
x2 = min(box[0] + box[2] / 2.0, 1) * image_width
y2 = min(box[1] + box[3] / 2.0, 1) * image_height
objectness = box[4]
for j in range((len(box) - 5) // 2):
cls_conf = box[5 + 2 * j]
cls_id = box[6 + 2 * j]
prob = objectness * cls_conf
batch_detections.append([cls_id, prob, x1, y1, x2, y2])
detections.append(batch_detections)
return detections
def training_data(self, groupStage):
print("(INFO) START TRAINING STAGE {} ! ".format(groupStage))
# Path to extracted feature
feature_path = os_path.join("./CUSTOMIZE_4_USER/MODEL_TRAINING",
groupStage, groupStage + ".npz")
model_path = os_path.join("./CUSTOMIZE_4_USER/MODEL_TRAINING",
groupStage, groupStage + ".pth")
data = np_load(feature_path)
feature = data['ColourFeature']
feature.astype(int)
label = data['Labels']
label = np_squeeze(label)
num_train = int(0.8 * len(feature))
num_test = len(feature) - num_train
arr = np_arange(len(feature))
shuffle(arr)
train_num = arr[:num_train]
test_num = arr[num_train:]
y_train = label[train_num]
y_test = label[test_num]
X_train = feature[train_num]
X_test = feature[test_num]
X_train = from_numpy(X_train).to(device).float()
y_train = from_numpy(y_train).to(device).long()
X_test = from_numpy(X_test).to(device).float()
y_test = from_numpy(y_test).to(device).long()
model = NeuralNet(input_size, hidden_size, num_classes).to(device)
# Loss and optimizer
criterion = CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=learning_rate)
max_acc = 30
for k in range(num_loop_epoch):
model.apply(self.weight_init)
for epoch in range(num_epochs):
# Forward pass
outputs = model(X_train)
loss = criterion(outputs, y_train)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (epoch + 1) % 1000 == 0:
print('Epoch [{}/{}], Loss: {:.4f}'.format(
epoch + 1, num_epochs, loss.item()))
with no_grad():
out_test = model(X_test)
_, predicted = torch_max(out_test.data, 1)
corrected = (predicted == y_test).sum().item()
total = len(y_test)
accuracy = 100 * (corrected / total)
print("accuracy: {} %".format(accuracy))
if accuracy > max_acc:
max_acc = accuracy
save(model.state_dict(), model_path)
print("Save this model " + str(groupStage) + ": " +
str(accuracy))
if max_acc == 30:
print("Bad training!!!")
def torch_max2(x, y):
return torch_max(torch_max(x, y[1])[0], y[0])[0]
def _ssd_discrete_metrics(self, predictions, targets, iou_threshold=0.5, is_cuda=False):
def __to_cuda(obj):
if is_cuda:
obj = obj.cuda()
return obj
predicted_boxes = predictions['boxes']
predicted_labels = predictions['labels']
predicted_class_scores = predictions['scores']
target_boxes = targets['boxes']
target_labels = targets['labels']
assert len(predicted_boxes) == len(predicted_labels) == len(predicted_class_scores) == len(
target_boxes) == len(target_labels)
target_images = list()
for i in range(len(target_labels)):
target_images.extend([i] * target_labels[i].size(0))
target_images = __to_cuda(LongTensor(target_images))
target_boxes = torch_cat(target_boxes, dim=0)
target_labels = torch_cat(target_labels, dim=0)
assert target_images.size(0) == target_boxes.size(0) == target_labels.size(0)
predicted_images = list()
for i in range(len(predicted_labels)):
predicted_images.extend([i] * predicted_labels[i].size(0))
predicted_images = __to_cuda(LongTensor(predicted_images))
predicted_boxes = torch_cat(predicted_boxes, dim=0)
predicted_labels = torch_cat(predicted_labels, dim=0)
predicted_class_scores = torch_cat(predicted_class_scores, dim=0)
assert predicted_images.size(0) == predicted_boxes.size(0) == predicted_labels.size(
0) == predicted_class_scores.size(0)
average_precisions = torch_zeros(self.num_classes, dtype=torch_float)
recalls = torch_zeros(self.num_classes, dtype=torch_float)
precisions = torch_zeros(self.num_classes, dtype=torch_float)
for c in range(self.num_classes):
target_class_images = target_images[target_labels == c]
target_class_boxes = target_boxes[target_labels == c]
total_objects = target_class_boxes.size(0)
target_class_boxes_detected = __to_cuda(torch_zeros(total_objects, dtype=torch_uint8))
class_c_predicted_images = predicted_images[predicted_labels == c]
class_c_predicted_boxes = predicted_boxes[predicted_labels == c]
class_c_predicted_class_scores = predicted_class_scores[predicted_labels == c]
class_c_num_detections = class_c_predicted_boxes.size(0)
if class_c_num_detections == 0:
continue
class_c_predicted_class_scores, sort_ind = torch_sort(class_c_predicted_class_scores, dim=0,
descending=True)
class_c_predicted_images = class_c_predicted_images[sort_ind]
class_c_predicted_boxes = class_c_predicted_boxes[sort_ind]
true_positives = __to_cuda(torch_zeros(class_c_num_detections, dtype=torch_float))
false_positives = __to_cuda(torch_zeros(class_c_num_detections, dtype=torch_float))
for d in range(class_c_num_detections):
this_detection_box = class_c_predicted_boxes[d].unsqueeze(0)
this_image = class_c_predicted_images[d]
object_boxes = target_class_boxes[target_class_images == this_image]
if object_boxes.size(0) == 0:
false_positives[d] = 1
continue
overlaps = find_jaccard_overlap(this_detection_box, object_boxes)
max_overlap, ind = torch_max(overlaps.squeeze(0), dim=0)
original_ind = LongTensor(range(target_class_boxes.size(0)))[target_class_images == this_image][ind]
if max_overlap.item() > iou_threshold:
if target_class_boxes_detected[original_ind] == 0:
true_positives[d] = 1
target_class_boxes_detected[original_ind] = 1
else:
false_positives[d] = 1
else:
false_positives[d] = 1
cumul_true_positives = torch_cumsum(true_positives, dim=0)
cumul_false_positives = torch_cumsum(false_positives, dim=0)
cumul_precision = cumul_true_positives / (cumul_true_positives + cumul_false_positives + 1e-10)
cumul_recall = cumul_true_positives / total_objects
recall_thresholds = [x / 10 for x in range(11)]
interpolated_precisions = __to_cuda(torch_zeros((len(recall_thresholds)), dtype=torch_float))
for i, threshold in enumerate(recall_thresholds):
recalls_above_threshold = cumul_recall >= threshold
if recalls_above_threshold.any():
interpolated_precisions[i] = cumul_precision[recalls_above_threshold].max()
else:
interpolated_precisions[i] = 0.
average_precisions[c] = interpolated_precisions.mean()
total_true_positives = torch_sum(true_positives)
recalls[c] = total_true_positives / max(float(total_objects), 1e-10)
precisions[c] = total_true_positives / max(
total_true_positives + torch_sum(false_positives), torch_tensor(1e-10))
return average_precisions.tolist(), recalls.tolist(), precisions.tolist()
def test(self,
epoch: int,
global_epoch: int,
config: LayerTrainingDefinition,
plot_every_n_epochs=1):
r"""
This method is the default test method for a horizontal autoencoder network block.
It operates on a given config of type LayerTrainingDefinition.
The method freezes the current layer and computes the test accuracy.
"""
test_loss = 0
test_acc = 0
alpha = config.tp_alpha
# switch to eval mode (dropout)
config.model.eval()
# execution times
t_start = time.time_ns()
with torch.no_grad():
img: Tensor = None
labels: Tensor = None
for data in self.test_loader:
img, label = data[0], data[1]
# copy all vars to device and calculate the topmost stack representation
# TODO: avoid calculating this representation twice (here and in forward())
dev_img = config.stack.upwards(img.to(self.device))
dev_label = label.to(self.device)
# ===================Forward=====================
decoding, prediction = config.model(dev_img)
loss_dc = self.decoding_criterion(decoding, dev_img)
loss_pred = self.pred_criterion(prediction, dev_label)
# Calculate Test Loss
test_loss += (alpha * loss_pred.item() +
loss_dc.item()) / (len(self.test_loader))
# Calculate Accuracy
_, predicted = torch_max(prediction.data, 1)
test_acc += (predicted.cpu().numpy() == label.numpy()
).sum() / (len(self.test_loader) * len(label))
self.test_losses.append(test_loss)
self.test_accs.append(test_acc)
t_start, t_delta = self.measure_time(t_start)
logging.info(
'Epoch [{}/{}] Test Loss:{:.4f} Test Acc:{:.4f} Time: {:.2f}'.
format(epoch + 1, config.num_epochs, test_loss, test_acc, t_delta))
self.writer.add_scalar('loss_total/test',
test_loss,
global_step=global_epoch)
self.writer.add_scalar(f'loss_{config.layer_name}/test',
test_loss,
global_step=epoch)
self.writer.add_scalar('accuracy_total/test',
test_acc,
global_step=global_epoch)
self.writer.add_scalar(f'accuracy_{config.layer_name}/test',
test_acc,
global_step=epoch)
def predict( # pylint: disable=too-many-arguments
self,
src_sequences: Tensor,
src_masks: Tensor,
tgt_bos_token: int,
decoding_method: str = 'greedy',
gpu_if_possible: bool = True) -> Tensor:
"""
Predict target token sequences from source token sequences.
"""
# selecting the device handling computations:
device = select_device(gpu_if_possible=gpu_if_possible)
# moving model parameters and buffers to such device:
self.model.to(device)
# moving inputs to such device:
src_sequences = src_sequences.to(device)
src_masks = src_masks.to(device)
# switching to inference mode:
self.model.eval()
if decoding_method == 'greedy':
# greedy decoding:
# computing encoder outputs, i.e. encoded representations of
# source tokens - from dimensionality (samples, tokens) to
# dimensionality (samples, tokens, features):
src_encoded_tokens = self.model.encode(src_tokens=src_sequences,
src_mask=src_masks)
# initializing predicted output sequences:
cumulative_tgt_sequences = torch_ones((1, 1), requires_grad=False)\
.fill_(value=tgt_bos_token).type_as(src_sequences)
# for each target position, the respective token is sequentially
# predicted, given the decoder auto-regressive predictive nature -
# for all sequences at the same time:
for _ in range(self.max_sequence_length - 1):
# computing logits - from dimensionality (samples, tokens,
# features) to dimensionality (samples, tokens, features):
next_token_logits = self.model.decode(
src_encoded_tokens=src_encoded_tokens,
src_mask=src_masks,
tgt_tokens=cumulative_tgt_sequences,
tgt_mask=allowed_positions_to_attend(
# positions to attend equal computed target tokens:
n_positions=cumulative_tgt_sequences.size(1)).to(
device))
# turning the logits of next (last) tokens in the sequences
# into log-probabilities - from dimensionality (samples,
# tokens, features) to dimensionality (samples, features):
next_token_log_probabilities = self.model.log_softmax_layer(
next_token_logits[:, -1] # next (last) tokens
)
# discretizing probabilities to predicted tokens - from
# dimensionality (samples, features) to dimensionality
# (samples):
next_tokens = torch_max(next_token_log_probabilities,
dim=1).indices[0]
# concatenating the newly predicted tokens to the sequences of
# already predicted tokens:
cumulative_tgt_sequences = torch_cat(
(cumulative_tgt_sequences, torch_ones(
(1, 1)).type_as(src_sequences).fill_(next_tokens)),
dim=1)
# FIXME: shapes not understood
# TODO: truncate the different predicted sequences in the
# mini-batch from their respective first padding token on
return cumulative_tgt_sequences
raise NotImplementedError("Unavailable decoding method: " +
decoding_method)
def sample(
self,
fc_feats,
att_feats,
att_masks=None,
sample_method=GREEDY,
temperature=1.0,
sample_n=1,
output_logsoftmax=1,
decoding_constraint=0,
block_trigrams=0,
remove_bad_endings=0,
):
#
# Beam search not implemented
# ===================
# beam_size=1,
# group_size=1,
# if beam_size > 1 and sample_method in ['greedy', 'beam_search']:
# return self._sample_beam(fc_feats, att_feats, att_masks, opt)
# if group_size > 1:
# return self._diverse_sample(fc_feats, att_feats, att_masks, opt)
batch_size = fc_feats.size(0)
state = self.init_hidden(batch_size * sample_n)
(
p_fc_feats,
p_att_feats,
pp_att_feats,
p_att_masks,
) = self.prepare_feature_for_generation(fc_feats, att_feats, att_masks)
#
# Don't implemnt diverse sampling
# ===================
# if sample_n > 1:
# p_fc_feats, p_att_feats, pp_att_feats, p_att_masks = repeat_tensors(
# sample_n, [p_fc_feats, p_att_feats, pp_att_feats, p_att_masks]
# )
# trigrams = [] # will be a list of batch_size dictionaries
#
# Sampled words
seq = fc_feats.new_zeros((batch_size * sample_n, self.seq_length),
dtype=torch_long)
#
# Log prob of words
seqLogprobs = fc_feats.new_zeros(batch_size * sample_n,
self.seq_length, self.vocab_size + 1)
for t in range(self.seq_length + 1):
#
# Beginning of sentence. No previous words sampled
if t == 0:
selected_word_indices = fc_feats.new_zeros(batch_size *
sample_n,
dtype=torch_long)
#
# Log prob of vocab
logprobs, state = self.get_logprobs_state(
selected_word_indices,
p_fc_feats,
p_att_feats,
pp_att_feats,
p_att_masks,
state,
output_logsoftmax=output_logsoftmax,
)
# if decoding_constraint and t > 0:
# tmp = logprobs.new_zeros(logprobs.size())
# tmp.scatter_(1, seq[:, t - 1].data.unsqueeze(1), float("-inf"))
# logprobs = logprobs + tmp
#
# Can't sample bad endings (is this right?)
if remove_bad_endings and t > 0:
tmp = logprobs.new_zeros(logprobs.size())
#
# See if bad ending was just chosen
prev_bad = isin(seq[:, t - 1].data.cpu().numpy(),
self.bad_endings_ix)
#
# If last chosen word was bad ending can't choose again?
# This doesn't seem like it will prevent a bad ending...
tmp[from_numpy(prev_bad.astype("uint8")), 0] = float("-inf")
logprobs = logprobs + tmp
#
# Trigrams not implemented
# ============
# if block_trigrams and t >= 3:
# # Store trigram generated at last step
# prev_two_batch = seq[:, t - 3 : t - 1]
# for i in range(batch_size): # = seq.size(0)
# prev_two = (
# prev_two_batch[i][0].item(),
# prev_two_batch[i][1].item(),
# )
# current = seq[i][t - 1]
# if t == 3: # initialize
# trigrams.append(
# {prev_two: [current]}
# ) # {LongTensor: list containing 1 int}
# elif t > 3:
# if prev_two in trigrams[i]: # add to list
# trigrams[i][prev_two].append(current)
# else: # create list
# trigrams[i][prev_two] = [current]
# # Block used trigrams at next step
# prev_two_batch = seq[:, t - 2 : t]
# mask = torch_zeros(
# logprobs.size(), requires_grad=False
# ).cuda() # batch_size x vocab_size
# for i in range(batch_size):
# prev_two = (
# prev_two_batch[i][0].item(),
# prev_two_batch[i][1].item(),
# )
# if prev_two in trigrams[i]:
# for j in trigrams[i][prev_two]:
# mask[i, j] += 1
# # Apply mask to log probs
# # logprobs = logprobs - (mask * 1e9)
# alpha = 2.0 # = 4
# logprobs = logprobs + (
# mask * -0.693 * alpha
# ) # ln(1/2) * alpha (alpha -> infty works best)
#
# Break if max length
if t == self.seq_length: # skip if we achieve maximum length
break
#
# Sample next word. Since this is a greedy approach without beams,
# simply take the most probable word
sampleLogprobs, selected_word_indices = torch_max(logprobs.data, 1)
selected_word_indices = selected_word_indices.view(-1).long()
#
# Stop when all sequences have finished.
# Encoded <EOS> as index 0.
if t == 0:
unfinished = selected_word_indices > 0
else:
#
# This prevents finished sequences from starting again
unfinished = unfinished * (selected_word_indices > 0)
#
# Don't allow restart of finished sequences
selected_word_indices = selected_word_indices * unfinished.type_as(
selected_word_indices)
#
# Update sentences
seq[:, t] = selected_word_indices
seqLogprobs[:, t] = logprobs
#
# Break if all are finished.
# If all false, then sum is 0.
if unfinished.sum() == 0:
break
return seq, seqLogprobs
def train_vgg_classifier(self, epoch: int, global_epoch: int,
config: LayerTrainingDefinition):
r"""
The vgg network has its own classification layer with a cost function that differs from the usual
autoencoder network block. Therefore it has its own training method.
"""
tot_t_dataload = 0
tot_t_loss = 0
tot_t_optim = 0
# switch to train mode (dropout)
config.model.train()
# _s means supervised _us unsupervised
iter_us = iter(self.unsupvised_loader)
iter_s = iter(self.supervised_loader)
n_batches = len(self.unsupvised_loader)
for ith_batch in range(n_batches):
t_start = time.time_ns()
img_s, label_s = (lambda d: (d[0], d[1]))(next(iter_s))
dev_img_s = img_s.to(self.device)
dev_label = label_s.to(self.device)
t_start, t_delta = self.measure_time(t_start)
tot_t_dataload += t_delta
# copy all vars to device and calculate the topmost stack representation
# TODO: avoid calculating this representation twice (here and in forward())
prediction = config.model(dev_img_s)
loss_pred = self.pred_criterion(prediction, dev_label)
t_start, t_delta = self.measure_time(t_start)
tot_t_loss += t_delta
# ===================backward====================
config.optimizer.zero_grad()
loss_pred.backward()
config.optimizer.step()
t_start, t_delta = self.measure_time(t_start)
tot_t_optim += t_delta
# ===================log========================
# Calculate Accuracy
_, predicted = torch_max(prediction.data, 1)
accuracy = (predicted.cpu().numpy()
== label_s.numpy()).sum() / len(label_s)
logging.info(
('Epoch [{}/{}] Train Loss:{:.4f} ' + 'Train Acc:{:.4f} ' +
'Time(Loading|Loss|Optim): {:.2f} {:.2f} {:.2f}').format(
epoch + 1, config.num_epochs, loss_pred.item(), accuracy,
tot_t_dataload, tot_t_loss, tot_t_optim))
self.writer.add_scalar('loss_linear/train',
loss_pred.item(),
global_step=global_epoch)
self.writer.add_scalar('accuracy_linear/train',
accuracy,
global_step=global_epoch)
def _build_targets(self, predictions, target_data, feature_map_width,
feature_map_height):
batch_size = target_data.size(0)
number_of_pixels = feature_map_height * feature_map_width
anchors_over_pixels = self.num_anchors * number_of_pixels
default_size = (batch_size, self.num_anchors, feature_map_height,
feature_map_width)
_1obj = torch_zeros(*default_size)
_1noobj = torch_ones(*default_size)
target_center_x_values = torch_zeros(*default_size)
target_center_y_values = torch_zeros(*default_size)
target_width_values = torch_zeros(*default_size)
target_height_values = torch_zeros(*default_size)
target_confidence_score_values = torch_zeros(*default_size)
target_class_values = torch_zeros(*default_size)
for image_index in range(batch_size):
start_index = image_index * anchors_over_pixels
end_index = (image_index + 1) * anchors_over_pixels
predicted_bounding_boxes = predictions[start_index:end_index].t()
ious = torch_zeros(anchors_over_pixels)
for t in range(self.max_object):
if target_data[image_index][t * 5 + 1] == -1:
break
ground_truth_center_x = target_data[image_index][
t * 5 + 1] * feature_map_width
ground_truth_center_y = target_data[image_index][
t * 5 + 2] * feature_map_height
ground_truth_width = target_data[image_index][
t * 5 + 3] * feature_map_width
ground_truth_height = target_data[image_index][
t * 5 + 4] * feature_map_height
ground_truth_bounding_boxes = FloatTensor([
ground_truth_center_x, ground_truth_center_y,
ground_truth_width, ground_truth_height
])
ground_truth_bounding_boxes = ground_truth_bounding_boxes.repeat(
anchors_over_pixels, 1).t()
ious = torch_max(
ious,
intersection_over_union(True,
predicted_bounding_boxes,
ground_truth_bounding_boxes,
is_corner_coordinates=False))
# https://github.com/marvis/pytorch-yolo2/issues/121#issuecomment-436388664
_1noobj[image_index][torch_reshape(ious, (
self.num_anchors, feature_map_height,
feature_map_width)) > self.ignore_threshold] = 0
for image_index in range(batch_size):
for t in range(self.max_object):
if target_data[image_index][t * 5 + 1] == -1:
break
anchor_index, ground_truth_width, ground_truth_height = self._find_most_matching_anchor(
feature_map_width, feature_map_height, image_index, t,
target_data)
ground_truth_center_x_pixel, ground_truth_center_y_pixel, ground_truth_bounding_box = \
self._compose_ground_truth_data(feature_map_width, feature_map_height, ground_truth_height,
ground_truth_width, image_index, t, target_data)
predicted_bounding_box = predictions[
image_index * anchors_over_pixels +
anchor_index * number_of_pixels +
ground_truth_center_y_pixel * feature_map_width +
ground_truth_center_x_pixel]
iou = intersection_over_union(False,
ground_truth_bounding_box,
predicted_bounding_box,
is_corner_coordinates=False)
_1obj[image_index][anchor_index][ground_truth_center_y_pixel][
ground_truth_center_x_pixel] = 1
_1noobj[image_index][anchor_index][
ground_truth_center_y_pixel][
ground_truth_center_x_pixel] = 0
target_center_x_values, target_center_y_values, target_width_values, target_height_values, \
target_confidence_score_values, target_class_values = self._set_target_values(
feature_map_width, feature_map_height, image_index, t, target_data, anchor_index, iou,
ground_truth_center_x_pixel, ground_truth_center_y_pixel, ground_truth_height, ground_truth_width,
target_center_x_values, target_center_y_values, target_class_values,
target_confidence_score_values, target_height_values, target_width_values)
return _1obj, _1noobj, target_center_x_values, target_center_y_values, target_width_values, \
target_height_values, target_confidence_score_values, target_class_values
def train(self, epoch: int, global_epoch: int,
config: LayerTrainingDefinition):
r"""
This method is the default train method for a horizontal autoencoder network block.
It takes care of dataset iteration and loss calculation / optimization.
It operates on a given config of type LayerTrainingDefinition.
This code represents what needs to be done in one epoch of training.
"""
tot_t_dataload = 0
tot_t_upstream = 0
tot_t_loss = 0
tot_t_optim = 0
# switch to train mode (dropout)
config.model.train()
# _s means supervised _us unsupervised
iter_us = iter(self.unsupvised_loader)
iter_s = iter(self.supervised_loader)
n_batches = len(self.unsupvised_loader)
for ith_batch in range(n_batches):
t_start = time.time_ns()
img_us, _ = (lambda d: (d[0], d[1]))(next(iter_us))
img_s, label_s = (lambda d: (d[0], d[1]))(next(iter_s))
dev_img_us = img_us.to(self.device)
dev_img_s = img_s.to(self.device)
dev_label = label_s.to(self.device)
t_start, t_delta = self.measure_time(t_start)
tot_t_dataload += t_delta
# copy all vars to device and calculate the topmost stack representation
# TODO: avoid calculating this representation twice (here and in forward())
with torch.no_grad():
dev_img_us = config.stack.upwards(dev_img_us)
dev_img_s = config.stack.upwards(dev_img_s)
t_start, t_delta = self.measure_time(t_start)
tot_t_upstream += t_delta
decoding_s, prediction = config.model(dev_img_s)
decoding_us, _ = config.model(dev_img_us)
loss_dc_s = self.decoding_criterion(decoding_s, dev_img_s)
loss_dc_us = self.decoding_criterion(decoding_us, dev_img_us)
loss_pred = self.pred_criterion(prediction, dev_label)
# tensor names: s = supervised, us = unsupervised, p = prediction(label)
# a = alpha r = regulation
loss_functions = {
'us*(1-a)+(s+p)*a+r':
lambda s, us, p, a, r: us * (1 - a) + (s + p) * a + r,
'(us+s)*(1-a)+p*a+r':
lambda s, us, p, a, r: (us + s) * (1 - a) + p * a + r,
'(us+10*s)*(1-a)+p*a+r':
lambda s, us, p, a, r: (us + 10 * s) * (1 - a) + p * a + r,
}
loss_f = loss_functions[config.ae_loss_function]
if isinstance(config.tp_alpha, nn.Parameter):
alpha = 0.5 + (torch.sigmoid(config.tp_alpha) - 0.5)
reg = -(torch.log(alpha) + torch.log(1 - alpha))
else:
alpha = config.tp_alpha
reg = 0
combo_loss = loss_f(loss_dc_s, loss_dc_us, loss_pred, alpha, reg)
t_start, t_delta = self.measure_time(t_start)
tot_t_loss += t_delta
# ===================backward====================
config.optimizer.zero_grad()
combo_loss.backward()
config.optimizer.step()
t_start, t_delta = self.measure_time(t_start)
tot_t_optim += t_delta
# ===================log========================
# Calculate Accuracy
_, predicted = torch_max(prediction.data, 1)
accuracy = (predicted.cpu().numpy()
== label_s.numpy()).sum() / len(label_s)
logging.info(
('Epoch [{}/{}] Train Loss:{:.4f} ' + 'Train Acc:{:.4f} ' +
'Time(Loading|Upstream|Loss|Optim): {:.2f} {:.2f} {:.2f} {:.2f}'
).format(epoch + 1, config.num_epochs, combo_loss.item(),
accuracy, tot_t_dataload, tot_t_upstream, tot_t_loss,
tot_t_optim))
if isinstance(config.tp_alpha, nn.Parameter):
self.writer.add_scalar('alpha/x',
config.tp_alpha.item(),
global_step=global_epoch)
self.writer.add_scalar('alpha/sig_x',
alpha.item(),
global_step=global_epoch)
self.writer.add_scalar('loss_total/train',
combo_loss.item(),
global_step=global_epoch)
self.writer.add_scalar(f'loss_{config.layer_name}/train',
combo_loss.item(),
global_step=epoch)
self.writer.add_scalar('accuracy_total/train',
accuracy,
global_step=global_epoch)
self.writer.add_scalar(f'accuracy_{config.layer_name}/train',
accuracy,
global_step=epoch)
def set_weights_scaled(
self,
weights: Tensor,
biases: Optional[Tensor] = None,
realistic: bool = False,
n_loops: int = 10,
weight_scaling_omega: Optional[float] = None) -> None:
r"""Set the tile weights (and biases) in a scaled fashion.
Similar to :meth:`set_weights`, however, additionally scales the weights
by a global scale :math:`\alpha`, that is then applied in digital at the
output of forward and backward pass, and the learning rate for this tile
is adjusted accordingly.
The weights are scaled by :math:`\omega/\max_{ij} |w_{ij}|` and the global
digital factor :math:`alpha` is set to :math:`\max_{ij} |w_{ij}|/\omega`.
It can be shown that such a constant factor greatly improves the SNR and
training accuracy as the full weight range of the analog devices are
used. See also `Rasch, Gokmen & Haensch (2019)`_ for more details.
Caution:
Using ``get_weights`` will now retrieve the true analog weights
*without* applying the global factor. To get the true weights, use
``get_weights`` and scale it by the :math:`\alpha` of this layer
which can be retrieved by ``get_alpha_scale()``.
Args:
weights: ``[out_size, in_size]`` weight matrix.
biases: ``[out_size]`` bias vector. This parameter is required if
``self.bias`` is ``True``, and ignored otherwise.
realistic: whether to use the forward and update pass to program the
weights iteratively, using :meth:`set_weights_realistic`.
n_loops: number of times the columns of the weights are set in a
closed-loop manner.
A value of ``1`` means that all columns in principle receive
enough pulses to change from ``w_min`` to ``w_max``.
weight_scaling_omega: where the weight max should be mapped in terms of
the weight range. Note that for ``omega`` larger than
the maximal weight of the device, weights will get
clipped for most devices. If this parameter is not
given, it will default to the ``weight_scaling_omega``
value set in the
:class:`~aihwkit.configs.utils.MappingParameter` of the
``rpu_config``
Returns:
None.
Raises:
ValueError: if the tile has bias but ``bias`` has not been
specified.
.. _`Rasch, Gokmen & Haensch (2019)`: https://arxiv.org/abs/1906.02698
"""
# Prepare the array expected by the pybind function, appending the
# biases row if needed.
weights_torch = weights.clone().detach().cpu()
if self.bias:
# Create a ``[out_size, in_size (+ 1)]`` matrix.
if biases is None:
raise ValueError('Analog tile has a bias, but no bias given')
biases_torch = unsqueeze(biases.clone().detach().cpu(), 1)
combined_weights = cat((weights_torch, biases_torch), dim=1)
else:
# Use only the ``[out_size, in_size]`` matrix.
combined_weights = weights_torch
mapping = self.rpu_config.mapping # type: ignore
omega = weight_scaling_omega
if omega is None:
omega = mapping.weight_scaling_omega
# Apply the scaling
if mapping.weight_scaling_omega_columnwise:
weight_max, _ = torch_max(abs(combined_weights), 1, keepdim=True)
else:
weight_max = torch_max(abs(combined_weights)).view(1)
if omega > 0:
alpha = weight_max / omega
elif mapping.learn_out_scaling_alpha:
alpha = ones_like(weight_max)
else:
alpha = None
if alpha is not None:
combined_weights = combined_weights / alpha
self.set_out_scaling_alpha(alpha)
# update the mapping field
self.rpu_config.mapping.weight_scaling_omega = omega # type: ignore
if realistic:
return self.tile.set_weights_realistic(combined_weights.numpy(),
n_loops)
return self.tile.set_weights(combined_weights.numpy())
def torch_max1(x, y):
return torch_max(x, y[0])
def detect_objects(self, image_as_tensor, min_score, max_overlap, top_k):
predicted_locs, predicted_scores = self.forward(image_as_tensor)
batch_size = predicted_locs.size(0)
n_priors = self.priors_cxcy.size(0)
predicted_scores = F.softmax(predicted_scores, dim=2)
all_images_boxes = list()
all_images_labels = list()
all_images_scores = list()
assert n_priors == predicted_locs.size(1) == predicted_scores.size(1)
for i in range(batch_size):
decoded_locs = cxcy_to_xy(
gcxgcy_to_cxcy(predicted_locs[i], self.priors_cxcy))
image_boxes = list()
image_labels = list()
image_scores = list()
for c in range(self.num_classes - 1):
class_scores = predicted_scores[i][:, c]
score_above_min_score = class_scores > min_score
n_above_min_score = score_above_min_score.sum().item()
if n_above_min_score == 0:
continue
class_scores = class_scores[score_above_min_score]
class_decoded_locs = decoded_locs[score_above_min_score]
class_scores, sort_ind = class_scores.sort(dim=0,
descending=True)
class_decoded_locs = class_decoded_locs[sort_ind]
overlap = find_jaccard_overlap(class_decoded_locs,
class_decoded_locs)
suppress = self._to_cuda(
torch_zeros((n_above_min_score), dtype=torch_uint8))
for box in range(class_decoded_locs.size(0)):
if suppress[box] == 1:
continue
suppress = torch_max(
suppress,
(overlap[box] > max_overlap).type(torch_uint8))
suppress[box] = 0
kept_indices = self._to_cuda(
suppress.type(BoolTensor).logical_not())
locs = class_decoded_locs[kept_indices].tolist()
for loc_index, loc in enumerate(locs):
locs[loc_index] = [
max(loc[0], 0.),
max(loc[1], 0.),
min(loc[2], 1.),
min(loc[3], 1.)
]
image_boxes.append(self._to_cuda(FloatTensor(locs)))
image_labels.append(
self._to_cuda(LongTensor(kept_indices.sum().item() * [c])))
image_scores.append(self._to_cuda(class_scores[kept_indices]))
if len(image_boxes) == 0:
image_boxes.append(
self._to_cuda(FloatTensor([[0., 0., 0., 0.]])))
image_labels.append(self._to_cuda(LongTensor([120])))
image_scores.append(self._to_cuda(FloatTensor([0.])))
image_boxes = self._to_cuda(torch_cat(image_boxes, dim=0))
image_labels = self._to_cuda(torch_cat(image_labels, dim=0))
image_scores = self._to_cuda(torch_cat(image_scores, dim=0))
n_objects = image_scores.size(0)
if n_objects > top_k:
image_scores, sort_ind = image_scores.sort(dim=0,
descending=True)
image_scores = image_scores[:top_k]
image_boxes = image_boxes[sort_ind][:top_k]
image_labels = image_labels[sort_ind][:top_k]
all_images_boxes.append(image_boxes)
all_images_labels.append(image_labels)
all_images_scores.append(image_scores)
return all_images_boxes, all_images_labels, all_images_scores
