def test_true_divide_out(self, device, dtype):
dividend = (torch.randn(5, device=device) * 100).to(dtype)
divisor = torch.arange(1, 6, device=device).to(dtype)
# Tests that requests for an integer quotient fail
if not dtype.is_floating_point:
integral_quotient = torch.empty(5, device=device, dtype=dtype)
with self.assertRaises(RuntimeError):
torch.true_divide(dividend, divisor, out=integral_quotient)
with self.assertRaises(RuntimeError):
torch.true_divide(dividend, 2, out=integral_quotient)
else:
# Tests that requests for a floating quotient succeed
floating_quotient = torch.empty(5, device=device, dtype=dtype)
div_result = dividend / divisor
self.assertEqual(
div_result,
torch.true_divide(dividend, divisor, out=floating_quotient))
self.assertEqual(
dividend / 2,
torch.true_divide(dividend, 2, out=floating_quotient))
torch.add(x, y, out=z1) print(z1) print(z1.shape) z2 = torch.add(x, y) print(z2) z3 = x + y print(z3) # subtraction z4 = x - y print(z4) # division z5 = torch.true_divide(x, y) print(z5) # multiplication z = x * y print(z) # inplace operations t = torch.zeros(3) t.add_(x) print(t) # exponentiation z = x.pow(2) print(z)
def decode(self, dec_state):
"""
decode
"""
long_tensor_type = torch.cuda.LongTensor if self.use_gpu else torch.LongTensor
b = dec_state.get_batch_size()
# [[0], [k*1], [k*2], ..., [k*(b-1)]]
self.pos_index = (long_tensor_type(range(b)) * self.k).view(-1, 1)
# Inflate the initial hidden states to be of size: (b*k, H)
dec_state = dec_state.inflate(self.k)
# Initialize the scores; for the first step,
# ignore the inflated copies to avoid duplicate entries in the top k
sequence_scores = long_tensor_type(b * self.k).float()
sequence_scores.fill_(-float('inf'))
sequence_scores.index_fill_(
0, long_tensor_type([i * self.k for i in range(b)]), 0.0)
# Initialize the input vector
input_var = long_tensor_type([self.BOS] * b * self.k)
# Store decisions for backtracking
stored_scores = list()
stored_predecessors = list()
stored_emitted_symbols = list()
for t in range(1, self.max_length + 1):
# Run the RNN one step forward
output, dec_state, attn = self.model.decode(input_var, dec_state)
log_softmax_output = output.squeeze(1)
# To get the full sequence scores for the new candidates, add the
# local scores for t_i to the predecessor scores for t_(i-1)
sequence_scores = sequence_scores.unsqueeze(1).repeat(1, self.V)
if self.length_average and t > 1:
sequence_scores = sequence_scores * \
(1 - 1/t) + log_softmax_output / t
else:
sequence_scores += log_softmax_output
scores, candidates = sequence_scores.view(b, -1).topk(self.k,
dim=1)
# Reshape input = (b*k, 1) and sequence_scores = (b*k)
input_var = (candidates % self.V)
sequence_scores = scores.view(b * self.k)
input_var = input_var.view(b * self.k)
# Update fields for next timestep
if torch.__version__ == '1.2.0':
predecessors = (candidates / self.V +
self.pos_index.expand_as(candidates)).view(
b * self.k)
else:
predecessors = (torch.true_divide(candidates, self.V) +
self.pos_index.expand_as(candidates)).view(
b * self.k).long()
dec_state = dec_state.index_select(predecessors)
# Update sequence scores and erase scores for end-of-sentence symbol so that they aren't expanded
stored_scores.append(sequence_scores.clone())
eos_indices = input_var.data.eq(self.EOS)
if eos_indices.nonzero(as_tuple=False).dim() > 0:
sequence_scores.data.masked_fill_(eos_indices, -float('inf'))
if self.ignore_unk:
# Erase scores for UNK symbol so that they aren't expanded
unk_indices = input_var.data.eq(self.UNK)
if unk_indices.nonzero(as_tuple=False).dim() > 0:
sequence_scores.data.masked_fill_(unk_indices,
-float('inf'))
# Cache results for backtracking
stored_predecessors.append(predecessors)
stored_emitted_symbols.append(input_var)
predicts, scores, lengths = self._backtrack(stored_predecessors,
stored_emitted_symbols,
stored_scores, b)
predicts = predicts[:, :1]
scores = scores[:, :1]
lengths = long_tensor_type(lengths)[:, :1]
mask = sequence_mask(lengths, max_len=self.max_length).eq(0)
predicts[mask] = self.PAD
return predicts, lengths, scores
#====================================================================# # Tensor Math and Comparison Operations # #====================================================================# x = torch.tensor([1,2,3]) y = torch.tensor([9,8,7]) # Addition z1 = torch.add(x,y) z2 = x+y # Subtraction z = x-y # Division z = torch.true_divide(x,y) # element-wise division if equal size # inplace operations t = torch.zeros(3) t.add_(x) t += x # Exponentiation z = x.pow(2) z = x**2 # Simple comparison z = x > 0 print(z) # Matrix multiplication
def softplus(x, sigma=1.):
y = torch.true_divide(x, sigma)
z = x.clone().float()
z[y < 34.0] = sigma * torch.log1p(torch.exp(y[y < 34.0]))
return z
def forward(self, grads, *args, **kwargs):
intrest = self.intrest_func(grads)
intrest_sum = intrest.abs().sum(dim=[1, 2, 3])
total = grads.abs().sum(dim=[1, 2, 3])
return torch.true_divide(intrest_sum, total)
# Averaging models
w_avg = copy.deepcopy(models[idx_user].state_dict())
# Record how many clients participated in this round of averaging.
n_participants = 1
for i in range(args.num_users):
if v_new[i] > v_old[i]:
version_matrix[idx_user, i] = v_new[i]
n_participants = n_participants + 1
w_model_to_merge = copy.deepcopy(models[i].state_dict())
n = len(w_avg.keys())
nt = 0
for key in w_avg.keys():
nt += 1
w_avg[key] = additive(w_avg[key], w_model_to_merge[key], args)
for key in w_avg.keys():
w_avg[key] = torch.true_divide(w_avg[key], n_participants)
print("Select user:"******", total number of participants:", n_participants, ", process:", r + 1, "/",
args.num_users)
global_model.load_state_dict(w_avg)
# Update local model versions for selected clients
version_matrix[idx_user, idx_user] = version_matrix[idx_user, idx_user] + 1
models[idx_user].load_state_dict(w_avg)
local_model = LocalUpdate(args=args, dataset=train_dataset,
idxs=user_groups[idx_user], logger=logger)
w, loss, t_model = local_model.update_weights(
model=copy.deepcopy(models[idx_user]), global_round=epoch)
# print global training loss every 'print_every' round
if (epoch + 1) % print_every == 0:
local_weights = []
for i in range(args.num_users):
def test_sparse_true_divide(self, device, dtype):
dividend = torch.randn(5, device=device).to(dtype)
divisor = 2
dividend_sparse = dividend.to_sparse()
casting_result = dividend.to(torch.get_default_dtype()) / 2
self.assertEqual(casting_result, torch.true_divide(dividend_sparse, 2).to_dense())
def error_loss(outputs, labels, mean_y, std_y):
loss = torch.mean(
torch.true_divide(
torch.abs((outputs * std_y + mean_y) - (labels * std_y + mean_y)),
(labels * std_y + mean_y)))
return loss
for dp in raw_eval:
eval_data.append([layer_structure + dp[0][0] + [dp[0][1]], dp[1][0]])
eval_data = np.array(eval_data)
eval_inputs = torch.tensor(
np.array([np.array(i) for i in eval_data[:, 0]], dtype='float32'))
eval_targets = torch.tensor(
np.array(eval_data[:, 1],
dtype='float32').reshape(eval_data.shape[0], 1))
net.load_state_dict(torch.load(ckpt_path))
net.eval()
eval_outputs = net(eval_inputs)
print(eval_outputs.data.numpy())
print(eval_targets)
print(
torch.mean(
torch.true_divide(torch.abs(eval_outputs - eval_targets),
eval_targets)).item())
exit()
pbar = tqdm(range(total_epochs))
border = "=" * 50
clear_border = _term_move_up() + "\r" + " " * len(border) + "\r"
for epoch in pbar: # loop over the dataset multiple times
running_loss = 0.0
for i, data in enumerate(train_loader, 0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
inputs = Variable(inputs, volatile=True).cuda()
labels = Variable(labels, volatile=True).cuda()
def compute(self):
"""
Aggregates and then calculates logs classification report similar to sklearn.metrics.classification_report.
Typically used during epoch_end.
Return:
aggregated precision, recall, f1, report
"""
total_examples = torch.sum(self.num_examples_per_class)
num_non_empty_classes = torch.nonzero(
self.num_examples_per_class).size(0)
precision = torch.true_divide(self.tp * 100,
(self.tp + self.fp + METRIC_EPS))
recall = torch.true_divide(self.tp * 100,
(self.tp + self.fn + METRIC_EPS))
f1 = torch.true_divide(2 * precision * recall,
(precision + recall + METRIC_EPS))
report = '\n{:50s} {:10s} {:10s} {:10s} {:10s}'.format(
'label', 'precision', 'recall', 'f1', 'support')
for i in range(len(self.tp)):
label = f'label_id: {i}'
if self.ids_to_labels and i in self.ids_to_labels:
label = f'{self.ids_to_labels[i]} ({label})'
report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format(
label, precision[i], recall[i], f1[i],
self.num_examples_per_class[i])
micro_precision = torch.true_divide(
torch.sum(self.tp) * 100,
torch.sum(self.tp + self.fp) + METRIC_EPS)
micro_recall = torch.true_divide(
torch.sum(self.tp) * 100,
torch.sum(self.tp + self.fn) + METRIC_EPS)
micro_f1 = torch.true_divide(
2 * micro_precision * micro_recall,
(micro_precision + micro_recall + METRIC_EPS))
macro_precision = torch.sum(precision) / num_non_empty_classes
macro_recall = torch.sum(recall) / num_non_empty_classes
macro_f1 = torch.sum(f1) / num_non_empty_classes
weighted_precision = torch.sum(
precision * self.num_examples_per_class) / total_examples
weighted_recall = torch.sum(
recall * self.num_examples_per_class) / total_examples
weighted_f1 = torch.sum(
f1 * self.num_examples_per_class) / total_examples
report += "\n-------------------"
report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format(
'micro avg', micro_precision, micro_recall, micro_f1,
total_examples)
report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format(
'macro avg', macro_precision, macro_recall, macro_f1,
total_examples)
report += ('\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format(
'weighted avg', weighted_precision, weighted_recall, weighted_f1,
total_examples) + '\n')
self.total_examples = total_examples
if self.mode == 'macro':
return macro_precision, macro_recall, macro_f1, report
elif self.mode == 'weighted':
return weighted_precision, weighted_recall, weighted_f1, report
elif self.mode == 'micro':
return micro_precision, micro_recall, micro_f1, report
elif self.mode == 'all':
return precision, recall, f1, report
else:
raise ValueError(
f'{self.mode} mode is not supported. Choose "macro" to get aggregated numbers \
or "all" to get values for each class.')
def __call__(self):
for epoch in range(10000):
total_loss = 0
for i, (img_data, label, box, landmarks) in enumerate(self.train_dataloader):
if torch.cuda.is_available() is True:
img_data = img_data.cuda()
gt_label = label.cuda()
gt_boxes = box.cuda()
gt_landmarks = landmarks.cuda()
else:
gt_label = label
gt_boxes = box
gt_landmarks = landmarks
pred_label, pred_offset, pred_landmarks = self.net(img_data)
pred_label = pred_label.view(-1, 2)
pred_offset = pred_offset.view(-1, 4)
pred_landmarks = pred_landmarks.view(-1, 10)
# print(pred_label.shape, pred_offset.shape, pred_landmarks.shape)
# print(label.shape, box.shape, landmarks.shape)
self.opt.zero_grad()
cls_loss = self.cls_loss(gt_label, pred_label)
box_loss = self.box_loss(gt_label, gt_boxes, pred_offset)
landmark_loss = self.landmark_loss(gt_label, gt_landmarks, pred_landmarks)
loss = cls_loss + box_loss + landmark_loss
loss.backward()
self.opt.step()
total_loss += loss.cpu().detach()
# self.summarywrite.add_scalars('train/loss', loss.cpu().detach().item(), global_step=self.epoch_num)
self.summarywrite.add_scalars("train/loss", {i: j for i, j in
zip(["loss", "cls_loss", "box_loss", "landmark_loss"],
[loss.cpu().detach().item(), cls_loss.cpu().item(),
box_loss.cpu().item(), landmark_loss.cpu().item()])
}, global_step=self.global_step)
self.global_step += 1
print(
f"epoch:{self.epoch_num}---loss:{loss.cpu().item()}---cls_loss:{cls_loss.cpu().item()}---box_loss:{box_loss.cpu().item()}---landmark_loss:{landmark_loss.cpu().item()}")
self.save_state_dict()
self.export_model(f"./param/{self.net_stage}.pt")
if self.epoch_num % 10 == 0:
with torch.no_grad():
for name, parmeter in self.net.named_parameters():
if parmeter.grad is not None:
avg_grad = torch.mean(parmeter.grad)
print(f"{name}----grad_avg:{avg_grad}")
self.summarywrite.add_scalar(f"grad_avg/{name}", avg_grad.item(), self.epoch_num)
self.summarywrite.add_histogram(f"grad/{name}", parmeter.cpu().numpy(), self.epoch_num)
if parmeter.data is not None:
avg_weight = torch.mean(parmeter.data)
print(f"{name}----weight_avg:{avg_weight}")
self.summarywrite.add_scalar(f"weight_avg/{name}", avg_weight.item(), self.epoch_num)
self.summarywrite.add_histogram(f"weight/{name}", parmeter.cpu().numpy(), self.epoch_num)
total = 0
right = 0
tp = 0
fp = 0
fn = 0
tn = 0
total_cls_loss = 0
total_box_loss = 0
total_landmark_loss = 0
for i, (img_data, label, box, landmarks) in enumerate(self.eval_dataloader):
if torch.cuda.is_available() is True:
img_data = img_data.cuda()
gt_label = label.cuda()
gt_boxes = box.cuda()
gt_landmarks = landmarks.cuda()
else:
gt_label = label
gt_boxes = box
gt_landmarks = landmarks
with torch.no_grad():
pred_label, pred_offset, pred_landmarks = self.net(img_data)
print(pred_label, pred_offset, pred_landmarks)
pred_label = pred_label.view(-1, 2)
pred_offset = pred_offset.view(-1, 4)
pred_landmarks = pred_landmarks.view(-1, 10)
total_cls_loss += self.cls_loss(gt_label, pred_label)
total_box_loss += self.box_loss(gt_label, gt_boxes, pred_offset)
total_landmark_loss += self.landmark_loss(gt_label, gt_landmarks, pred_landmarks)
pred_label = torch.argmax(pred_label, dim=1)
mask = gt_label <= 1
right += torch.sum(gt_label[mask] == pred_label[mask])
total += gt_label[mask].shape[0]
p_mask = gt_label == 1
tp += torch.sum(gt_label[p_mask] == pred_label[p_mask])
fp += torch.sum(gt_label[p_mask] != pred_label[p_mask])
n_mask = gt_label == 0
tn += torch.sum(gt_label[n_mask] == pred_label[n_mask])
fn += torch.sum(gt_label[n_mask] != pred_label[n_mask])
# acc = right.cpu().detach() / total
acc = torch.true_divide(right, total)
# precision = tp / (tp + fp)
precision = torch.true_divide(tp, (tp + fp))
# recall = tp / (tp + fn)
recall = torch.true_divide(tp, (tp + fn))
# f1 = 2 * precision * recall / (precision + recall)
f1 = torch.true_divide((2 * precision * recall), (precision + recall))
avg_cls_loss = total_cls_loss / i
avg_box_loss = total_box_loss / i
avg_lanmark_loss = total_landmark_loss / i
self.summarywrite.add_scalars("eval/loss", {i: j for i, j in
zip(["avg_cls_loss", "avg_box_loss", "avg_lanmark_loss"],
[avg_cls_loss.cpu().item(),
avg_box_loss.cpu().item(), avg_lanmark_loss.cpu().item()])
}, global_step=self.epoch_num)
self.summarywrite.add_scalars("eval_set", {
i: j for i, j in
zip(["acc", "precision", "recall", "f1"], [acc, precision, recall, f1])
}, global_step=self.epoch_num)
print("Epoch %d, " % self.epoch_num,
f"result on eval set: acc {acc.cpu().item()}",
f"precision {precision.cpu().item()}",
f"recall {recall.cpu().item()}",
f"f1 {f1.cpu().item()}")
self.epoch_num += 1
colorimg[y, x, :] = colors[2]
elif index == 1:
colorimg[y, x, :] = colors[1]
elif index == 3:
colorimg[y, x, :] = colors[3]
else:
colorimg[y, x, :] = colors[0]
# colorimg[y,x,:] = np.mean(selected_colors, axis=0)
return colorimg.astype(np.uint8)
for idx, data in enumerate(val_loader):
image, mask = data
input = torch.true_divide(image, 255).to(torch.float).to(device)
target = mask.to(torch.long).to(device)
output = model(input)
output = softmax(output).squeeze(dim=0).detach().cpu()
rgb_out = masks_to_colorimg(output.numpy())
os.mkdir(f'{OUTPUT_FOLDER}/sample_{idx}')
io.imsave(f'{OUTPUT_FOLDER}/sample_{idx}/INPUT.png',
image.squeeze(0).permute(1, 2, 0).numpy())
io.imsave(f'{OUTPUT_FOLDER}/sample_{idx}/GT.png',
masks_to_colorimg(mask.squeeze(0).numpy()))
io.imsave(f'{OUTPUT_FOLDER}/sample_{idx}/pred.png', rgb_out)
if idx > 4:
break
two_pi = torch.tensor(2 * np.pi, dtype=torch.float).cuda()
tVec = torch.arange(0, sequenceDuration, 1 / fs, dtype=torch.float).cuda()
chripDurations = (minChirpDuration +
torch.mul(torch.rand(batch_size, dtype=torch.float),
maxChirpDuration - minChirpDuration)).cuda() # sec
startFreqs = torch.mul(torch.rand(batch_size, dtype=torch.float),
fs / 2).cuda() # [hz/sec]
startPhases = torch.mul(torch.rand(batch_size, dtype=torch.float),
two_pi).cuda() # [rad/sec]
k = torch.pow(
4000 / 100, 1 / chripDurations
) # k is the rate of exponential change in frequency to transform from 100 to 4000 hz in 10 seconds
freqFactor = torch.true_divide(
torch.pow(k[None, :].repeat(tVec.shape[0], 1), tVec[:, None].repeat(
1, k.shape[0])) - 1,
torch.log(k)[None, :].repeat(tVec.shape[0], 1))
phases = torch.mul(
torch.mul(startFreqs[None, :].repeat(tVec.shape[0], 1), two_pi),
freqFactor) + startPhases[None, :].repeat(tVec.shape[0], 1)
pureSinWaves = torch.sin(phases)
noise = torch.mul(torch.randn_like(pureSinWaves), noiseStd)
noisySinWaves = pureSinWaves + noise
modelInputMean_dbW = 10 * np.log10(
np.mean(np.power(noisySinWaves.cpu().numpy(), 2)))
'''
plt.plot(tVec.cpu().numpy(), noisySinWaves[:, 0].cpu().numpy())
plt.xlabel('sec')
plt.title('LSTM input wave')
def train_net_fednova(net_id,
net,
global_model,
train_dataloader,
test_dataloader,
epochs,
lr,
args_optimizer,
device="cpu"):
logger.info('Training network %s' % str(net_id))
train_acc = compute_accuracy(net, train_dataloader, device=device)
test_acc, conf_matrix = compute_accuracy(net,
test_dataloader,
get_confusion_matrix=True,
device=device)
logger.info('>> Pre-Training Training accuracy: {}'.format(train_acc))
logger.info('>> Pre-Training Test accuracy: {}'.format(test_acc))
optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()),
lr=lr,
momentum=args.rho,
weight_decay=args.reg)
criterion = nn.CrossEntropyLoss().to(device)
if type(train_dataloader) == type([1]):
pass
else:
train_dataloader = [train_dataloader]
#writer = SummaryWriter()
tau = 0
for epoch in range(epochs):
epoch_loss_collector = []
for tmp in train_dataloader:
for batch_idx, (x, target) in enumerate(tmp):
x, target = x.to(device), target.to(device)
optimizer.zero_grad()
x.requires_grad = True
target.requires_grad = False
target = target.long()
out = net(x)
loss = criterion(out, target)
loss.backward()
optimizer.step()
tau = tau + 1
epoch_loss_collector.append(loss.item())
epoch_loss = sum(epoch_loss_collector) / len(epoch_loss_collector)
logger.info('Epoch: %d Loss: %f' % (epoch, epoch_loss))
a_i = (tau - args.rho * (1 - pow(args.rho, tau)) /
(1 - args.rho)) / (1 - args.rho)
global_model_para = global_model.state_dict()
net_para = net.state_dict()
norm_grad = copy.deepcopy(global_model.state_dict())
for key in norm_grad:
#norm_grad[key] = (global_model_para[key] - net_para[key]) / a_i
norm_grad[key] = torch.true_divide(
global_model_para[key] - net_para[key], a_i)
train_acc = compute_accuracy(net, train_dataloader, device=device)
test_acc, conf_matrix = compute_accuracy(net,
test_dataloader,
get_confusion_matrix=True,
device=device)
logger.info('>> Training accuracy: %f' % train_acc)
logger.info('>> Test accuracy: %f' % test_acc)
logger.info(' ** Training complete **')
return train_acc, test_acc, a_i, norm_grad
def binary_dice(pred: Tensor, target: Tensor, eps=1e-9) -> torch.float:
"""NOTE: Must copy to CPU, so expensive. Should not be peformed every global step."""
p, t = pred.cpu().detach().view(-1), target.cpu().detach().view(-1)
intersect = (t * p).sum()
return torch.true_divide(2 * intersect, t.sum() + p.sum() + eps)
def prediction_loop(self,
dataloader: DataLoader,
description: str,
prediction_loss_only: Optional[bool] = None,
extract_path: Optional[str] = None,
cache_path: Optional[str] = None) -> PredictionOutput:
"""
Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`.
Works both with or without labels.
"""
prediction_loss_only = (prediction_loss_only
if prediction_loss_only is not None else
self.args.prediction_loss_only)
model = self.model
# multi-gpu eval
if self.args.n_gpu > 1:
model = torch.nn.DataParallel(model)
else:
model = self.model
# Note: in torch.distributed mode, there's no point in wrapping the model
# inside a DistributedDataParallel as we'll be under `no_grad` anyways.
batch_size = dataloader.batch_size
eval_losses: List[float] = []
hidden_states: torch.tensor = None
preds: torch.Tensor = None
label_ids: torch.Tensor = None
model.eval()
if self.args.past_index >= 0:
self._past = None
# Unfortunate, but we'll run through the dataloader once to count the number of tokens (or this could be pre-processed)
if extract_path is not None:
stimulus_mask = lambda tokens: (tokens != 101) & (tokens != 102
) & (tokens != 0)
cached_masks = None
if osp.exists(f"{cache_path}.npy"):
# np instead of torch, something's funky with Vivek's env.
cached_masks = torch.from_numpy(np.load(f"{cache_path}.npy"))
else:
all_masks = None
limit_tokens = self.custom_cfg.TASK.EXTRACT_TOKENS_LIMIT
# Calculate the random ratio of tokens to grab (we specify number of tokens to extract)
total_tokens = 0
for inputs in dataloader:
tokens = inputs["input_ids"]
total_tokens += stimulus_mask(tokens).sum()
subset_ratio = torch.true_divide(limit_tokens, total_tokens)
# Seed, we want to be sure that we're finding the same stimuli
disable_tqdm = not self.is_local_process_zero(
) or self.args.disable_tqdm
samples_count = 0
for inputs in tqdm(dataloader, desc=description, disable=disable_tqdm):
loss, logits, labels, states = self.prediction_step(
model,
inputs,
prediction_loss_only,
output_hidden_states=extract_path is not None)
batch_size = inputs[list(inputs.keys())[0]].shape[0]
if loss is not None:
eval_losses.append(loss * batch_size)
if states is not None:
# L + 1 [ Batch x Length x Hidden ] (layers and embedding)
if cached_masks is not None:
cached_masks = cached_masks.to(logits.device)
mask = cached_masks[samples_count:samples_count +
inputs["input_ids"].shape[0]] # B x T
mask = mask[:, :inputs["input_ids"].
shape[1]] # Dynamic padding
else:
subset_mask = torch.full(inputs["input_ids"].shape,
subset_ratio,
device=logits.device)
mask = (torch.bernoulli(subset_mask).long() &
stimulus_mask(inputs["input_ids"])).bool() # B X T
if all_masks is None:
all_masks = mask
else:
all_masks = nested_concat(all_masks,
mask,
padding_index=-100) # B x T
# [1:] to drop embedding layer
states = torch.stack(states)[1:].permute(1, 2, 0,
3) # B x T x L x H
target_tokens = states[mask] # M x L x H
if hidden_states is None:
hidden_states = target_tokens
else:
hidden_states = torch.cat([hidden_states, target_tokens],
dim=0)
samples_count += batch_size
if logits is not None:
preds = logits if preds is None else nested_concat(
preds, logits, padding_index=-100)
if labels is not None:
label_ids = labels if label_ids is None else nested_concat(
label_ids, labels, padding_index=-100)
if extract_path is not None:
os.makedirs(osp.split(extract_path)[0], exist_ok=True)
np.save(extract_path,
hidden_states.half().cpu().numpy()) # half to save memory
if cached_masks is None:
os.makedirs(osp.split(cache_path)[0], exist_ok=True)
np.save(cache_path, all_masks.cpu().numpy())
if self.args.past_index and hasattr(self, "_past"):
# Clean the state at the end of the evaluation loop
delattr(self, "_past")
if self.args.local_rank != -1:
# In distributed mode, concatenate all results from all nodes:
if preds is not None:
preds = self.distributed_concat(
preds, num_total_examples=self.num_examples(dataloader))
if label_ids is not None:
label_ids = self.distributed_concat(
label_ids,
num_total_examples=self.num_examples(dataloader))
# Finally, turn the aggregated tensors into numpy arrays.
if preds is not None:
preds = preds.cpu().numpy()
if label_ids is not None:
label_ids = label_ids.cpu().numpy()
if self.compute_metrics is not None and preds is not None and label_ids is not None:
metrics = self.compute_metrics(
EvalPrediction(predictions=preds, label_ids=label_ids))
else:
metrics = {}
if len(eval_losses) > 0:
metrics["eval_loss"] = np.sum(eval_losses) / samples_count
# Prefix all keys with eval_
for key in list(metrics.keys()):
if not key.startswith("eval_"):
metrics[f"eval_{key}"] = metrics.pop(key)
return PredictionOutput(predictions=preds,
label_ids=label_ids,
metrics=metrics)
def test_true_divide(self, device, dtype):
dividend = torch.randn(5, device=device).to(dtype)
divisor = torch.arange(1, 6, device=device).to(dtype)
casting_result = dividend.to(torch.get_default_dtype()) / divisor.to(
torch.get_default_dtype())
self.assertEqual(casting_result, torch.true_divide(dividend, divisor))
def _true_divide(dividend, divisor):
return torch.true_divide(dividend, divisor)
def forward(self, x, y):
return torch.true_divide(x, y)
def train_model(model, optimizer, scheduler, num_epochs=20):
best_model_wts = copy.deepcopy(model.state_dict())
best_loss = 1e10
patience = 0
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
since = time.time()
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
scheduler.step()
for param_group in optimizer.param_groups:
print("LR", param_group['lr'])
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
metrics = defaultdict(float)
epoch_samples = 0
for inputs, labels in dataloaders[phase]:
inputs = torch.true_divide(inputs, 255)
inputs = inputs.type(torch.float)
labels = labels.type(torch.float)
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
loss = calc_loss(outputs, labels, metrics)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
epoch_samples += inputs.size(0)
print_metrics(metrics, epoch_samples, phase)
epoch_loss = metrics['loss'] / epoch_samples
# deep copy the model
if phase == 'val' and epoch_loss < best_loss:
print("saving best model")
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
patience = 0
elif phase == 'val':
patience += 1
time_elapsed = time.time() - since
print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
if patience >= 15:
print("out of patience breaking the training loop :(")
break
print('Best val loss: {:4f}'.format(best_loss))
# load best model weights
model.load_state_dict(best_model_wts)
return model
def forward(self, x, y):
# Add transpose to hide shape/type information
# Otherwise shape and type are still avaiable from input.
x = x.transpose(1, 2)
y = y.transpose(1, 2)
return torch.true_divide(x, y)
def nnet(train_set, test_set, train_labels, test_labels, classes):
# 1. Shuffle train and test set
train_set = np.transpose(train_set, (0, 3, 1, 2))
test_set = np.transpose(test_set, (0, 3, 1, 2))
train_set = train_set.astype('float32')
test_set = test_set.astype('float32')
#print('Dataset loaded.')
indices = np.arange(train_set.shape[0])
np.random.shuffle(indices)
train_set = train_set[indices]
train_labels = train_labels[indices]
indices = np.arange(test_set.shape[0])
np.random.shuffle(indices)
test_set = test_set[indices]
test_labels = test_labels[indices]
# 2. Transform train and test set into tensors
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=batch_size,
shuffle=False)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=batch_size,
shuffle=False)
train_labels = torch.tensor(train_labels, dtype=torch.long)
test_labels = torch.tensor(test_labels, dtype=torch.long)
# Use torch.true_divide(image, 255) insted
#transform = tr.Normalize((128, 128, 128), (127, 127, 127))
# 3. Create a model
net = Net()
# 4. Define a loss function and the optimizer
criterion = nn.CrossEntropyLoss() # or MSELoss
optimizer = torch.optim.SGD(net.parameters(),
lr=learning_rate,
momentum=0.9)
#optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
# 5. Train the network (if necessary)
net.train()
if os.path.isfile('.pth'):
net.load_state_dict(torch.load('.pth'))
else:
for epoch in range(epochs): # loop over the dataset multiple times
running_loss = 0.0
for i, inputs in enumerate(train_loader):
curr_size = inputs.size()[0]
for j in range(curr_size):
inputs[j] = torch.true_divide(inputs[j], 255)
outputs = net(inputs)
#right_prob = torch.zeros([curr_size, n_classes], dtype=torch.float32)
#for j in range(curr_size):
# right_prob[j, int(train_labels[i*batch_size+j])] = 1
loss = criterion(
outputs, train_labels[i * batch_size:(i + 1) * batch_size])
loss.backward()
optimizer.step()
optimizer.zero_grad()
# Print statistics
running_loss += loss.item()
if i % 100 == 0:
print('[%d, %5d] loss: %.3f' %
(epoch + 1, i + 1, running_loss / 100))
running_loss = 0.0
print('Training ended.')
# Save the obtained model
torch.save(net.state_dict(), 'trained_network.pth')
# 6. Test the network
net.eval() # disable batch normalization
cmc = np.zeros((n_classes, n_classes))
correct = 0
total = 0
total_predicted = []
with torch.no_grad():
for i, inputs in enumerate(test_loader):
curr_size = inputs.size()[0]
for j in range(curr_size):
inputs[j] = torch.true_divide(inputs[j], 255)
outputs = net(inputs)
predicted = torch.argmax(outputs, dim=1)
total_predicted.extend(predicted)
total += curr_size
correct += (predicted == test_labels[i * batch_size:(i + 1) *
batch_size]).sum().item()
for predict, test_label in zip(
predicted,
test_labels[i * batch_size:(i + 1) * batch_size]):
cmc[int(test_label), int(predict)] += 1.0
#if i % 10 == 0:
#print(predicted)
#print(test_labels[i*batch_size:(i+1)*batch_size], end='\n')
precision = []
recall = []
for i in range(n_classes):
if cmc[i, i] != 0:
precision.append(cmc[i, i] / np.sum(cmc[:, i]))
recall.append(cmc[i, i] / np.sum(cmc[i, :]))
precision = np.mean(np.asarray(precision))
recall = np.mean(np.asarray(recall))
print('Accuracy of the network on the test images: {0:.2f} %'.format(
100 * correct / total))
print('Classifier\'s mean precision: ' + "{0:.2f}".format(precision))
print('Classifier\'s mean recall: ' + "{0:.2f}".format(recall))
# 7. Test performance for every class
class_correct = list(0. for i in range(n_classes))
class_total = list(0. for i in range(n_classes))
with torch.no_grad():
for i, inputs in enumerate(test_loader):
curr_size = inputs.size()[0]
for j in range(curr_size):
inputs[j] = torch.true_divide(inputs[j], 255)
outputs = net(inputs)
predicted = torch.argmax(outputs, dim=1)
c = (predicted == test_labels[i * batch_size:(i + 1) *
batch_size]).squeeze()
for j in range(curr_size):
label = test_labels[i * batch_size + j]
class_correct[int(label)] += c[j].item()
class_total[int(label)] += 1
print()
for i in range(n_classes):
print('Accuracy of %5s: %2d %%' %
(classes[i], 100 * class_correct[i] / class_total[i]))
print()
return total_predicted
def do_train():
mean, std, count = statistic(args.train_path)
# print("Mean: {}, Standard deviation: {}, count: {}".format(mean, std, count))
custom_transform = transforms.Compose([
transforms.Resize([args.size, args.size]),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
total = sum(count)
weight = []
for number in count:
weight += [torch.true_divide(total, number)] * number
# print("Check weight length: ", len(weight))
# print("Check total length: ", int(total))
sampler = torch.utils.data.sampler.WeightedRandomSampler(
weights=torch.Tensor(weight), num_samples=int(total))
del count, weight
train_set = ImageFolder(root=args.train_path, transform=custom_transform)
val_set = ImageFolder(root=args.validation_path,
transform=custom_transform)
train_loader = DataLoader(train_set, batch_size=args.bs, \
pin_memory=True, drop_last=True, sampler=sampler)
val_loader = DataLoader(val_set, batch_size=args.bs)
if args.loadOrNot == 1:
model = torch.load(args.model_path + args.model_load)
else:
model = Classifier().cuda()
LossFunction = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.05)
# optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
# optimizer = torch.optim.SGD(
# model.parameters(), lr=args.lr, momentum=0.9)
BestAccuracy = 0.0
if args.model_save:
torch.save(model, args.model_path + args.model_save)
for epoch in range(args.ep):
model.train()
for i, data in enumerate(train_loader):
optimizer.zero_grad()
train_pred = model(data[0].cuda())
batch_loss = LossFunction(train_pred, data[1].cuda())
batch_loss.backward()
optimizer.step()
train_loss = 0.0
train_acc = 0.0
model.eval()
for i, data in enumerate(train_loader):
train_pred = model(data[0].cuda())
batch_loss = LossFunction(train_pred, data[1].cuda())
train_acc += np.sum(
np.argmax(train_pred.cpu().data.numpy(), axis=1) ==
data[1].numpy())
train_loss += batch_loss.item()
val_acc = 0.0
model.eval()
for i, data in enumerate(val_loader):
val_pred = model(data[0].cuda())
val_acc += np.sum(
np.argmax(val_pred.cpu().data.numpy(), axis=1) ==
data[1].numpy())
accuracy = val_acc / len(val_set)
print('Epoch: %d, Train Acc: %3.6f Loss: %3.6f Val Acc: %3.6f' %
(epoch, train_acc / len(train_set), train_loss / len(train_set),
accuracy))
if args.model_save and accuracy > BestAccuracy:
BestAccuracy = accuracy
torch.save(model, args.model_path + args.model_save)
def lif_j(j, tau_ref, tau_rc, amplitude=1.):
j = torch.true_divide(1., j)
j = torch.log1p(j)
j = tau_ref + tau_rc * j
j = torch.true_divide(amplitude, j)
return j
loss.backward()
optimizer.step()
# GTLVQ uses projected SGD, which means to orthogonalize the subspaces after every gradient update.
model.gtlvq.orthogonalize_subspace()
if batch_idx % log_interval == 0:
acc = calculate_prototype_accuracy(distances, y_train, plabels)
print(
f"Epoch: {epoch + 1:02d}/{num_epochs:02d} Epoch Progress: {100. * batch_idx / len(train_loader):02.02f} % Loss: {loss.item():02.02f} \
Train Acc: {acc.item():02.02f}")
# Test
with torch.no_grad():
model.eval()
correct = 0
total = 0
for x_test, y_test in test_loader:
x_test, y_test = x_test.to(device), y_test.to(device)
test_distances = model(torch.tensor(x_test))
test_plabels = model.gtlvq.cls.prototype_labels.to(device)
i = torch.argmin(test_distances, 1)
correct += torch.sum(y_test == test_plabels[i])
total += y_test.size(0)
print("Accuracy of the network on the test images: %d %%" %
(torch.true_divide(correct, total) * 100))
# Save the model
PATH = "./glvq_mnist_model.pth"
torch.save(model.state_dict(), PATH)
def get_accuracy(logit, true_y):
pred_y = torch.argmax(logit, dim=1)
return torch.true_divide((pred_y == true_y).sum(), len(true_y))
def test_implementation_des(cfg):
"""
Simulates a train and val epoch to check if the gradients are being updated,
metrics are being calculated correctly
Args:
cfg (CfgNode): configs. Details can be found in
slowfast/config/defaults.py
"""
# Set random seed from configs.
np.random.seed(cfg.RNG_SEED)
torch.manual_seed(cfg.RNG_SEED)
# Setup logging format.
logging.setup_logging(cfg.OUTPUT_DIR)
# Print config.
logger.info("Test implementation")
# Build the video model and print model statistics.
model = build_clevrer_model(cfg)
# Construct the optimizer.
optimizer = optim.construct_optimizer(model, cfg)
start_epoch = cu.load_train_checkpoint(cfg, model, optimizer)
# Create the video train and val loaders.
if cfg.TRAIN.DATASET != 'Clevrer_des':
print("This train script does not support your dataset: -{}-. Only Clevrer_des".format(cfg.TRAIN.DATASET))
exit()
train_loader = build_dataloader(cfg, "train")
val_loader = build_dataloader(cfg, "val")
# Create meters.
train_meter = ClevrerTrainMeter(len(train_loader), cfg)
val_meter = ClevrerValMeter(len(val_loader), cfg)
# Perform the training loop.
logger.info("Start epoch: {}".format(start_epoch + 1))
# Train for one epoch.
model_before = copy.deepcopy(model)
cur_epoch = start_epoch
train_epoch(
train_loader, model, optimizer, train_meter, cur_epoch, cfg, test_imp=True
)
print("Check how much parameters changed")
for (p_b_name, p_b), (p_name, p) in zip(model_before.named_parameters(), model.named_parameters()):
if p.requires_grad:
print("Parameter requires grad:")
print(p_name, p_b_name)
#Calculate ratio of change
change = torch.abs(torch.norm(p) - torch.norm(p_b))
print("Ratio of change = {}".format(torch.true_divide(change, torch.norm(p_b))))
if (p_b != p).any():
print("--Check--")
else:
print("ALERT - WEIGHTS DID NOT CHANGE WITH TRAINING.")
else:
print("Parameter does not require grad:")
print(p_name)
print(p)
print("Val epoch")
eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, test_imp=True)
print(2)
dataLoader = torch.utils.data.DataLoader(dataset=dataset,
batch_size=batch_size,
num_workers=2,
shuffle=True)
optim = torch.optim.Adam(net.parameters(), lr=0.1, betas=(0.5, 0.999))
fixed_x, fixed_label = dataset.get_fixed()
fixed_x = torch.tensor(fixed_x)
fixed_label = torch.tensor(fixed_label)
print(3)
for epoch in range(1):
for i, data in tqdm(enumerate(dataLoader, 0)):
net.zero_grad()
x = Variable(data[0])
label = Variable(data[1])
y = net(x)
loss = torch.nn.BCELoss()(y, label)
optim.zero_grad()
loss.backward()
optim.step()
print('loss: ', loss.mean())
test_y = net(x)
real = torch.argmax(y, dim=1)
predict = torch.argmax(test_y, dim=1)
acc = torch.true_divide(torch.sum(predict == real), len(real))
print('acc: ', acc)
val_lo = (val_total_loss / len(valiter))
val_acc = [val_accuracy[i] / len(valiter) for i in range(2)]
print(f'Epoch {epoch}: train_loss: {train_lo:.4f} f1_macro: {train_acc[0]} f1_micro {train_acc[1]} | val_loss: {val_lo:.4f} f1_macro: {val_acc[0]} f1_micro {val_acc[1]}')
return model
if __name__ == '__main__':
cuda = torch.cuda.is_available()
device = torch.device("cpu") if not cuda else torch.device("cuda:0")
dataL = dataload()
text_field, label_field, trainds, valds, vectors = dataL.buildVocab()
values = [label_field.vocab.freqs.most_common(15)[i][1] for i in range(15)]
weight = torch.true_divide(torch.tensor(values), sum(values))
crit = nn.CrossEntropyLoss(weight=weight.cuda())
batch_size = 64
traindl, valdl = data.BucketIterator.splits(datasets=(trainds, valds),
batch_size=batch_size,
sort_key=lambda x: len(x.text),
device=device,
sort_within_batch=True,
repeat=False)
# print("[Corpus]: train: {}, test: {}, vocab: {}, labels: {}".format(
# len(train_iter.dataset), len(val_iter.dataset), len(TEXT.vocab), len(LABEL.vocab)))
train_iter, val_iter = traindl, valdl
TEXT, LABEL = text_field, label_field
ntokens, nlabels = len(TEXT.vocab), len(LABEL.vocab)
