def _obtain_labels_(self,
score_arc: torch.tensor,
score_rel: torch.tensor) -> Tuple[List[List[int]],
List[List[str]]]:
arc_prediction: torch.tensor = score_arc.argmax(-1)
relation_prediction: torch.tensor = score_rel.argmax(-1)
relation_prediction = relation_prediction.gather(-1, arc_prediction.unsqueeze(-1)).squeeze(-1)
arc_prediction = [[arc+1 if token_index != arc else 0 for token_index, arc in enumerate(batch)]
for batch in arc_prediction]
relation_prediction = [[self.relations_dictionary.get_item_for_index(rel_tag_idx)
for rel_tag_idx in batch] for batch in relation_prediction]
return arc_prediction, relation_prediction
def make_output(self, input_: torch.tensor, target: torch.tensor):
''' target - one hot for main task
return output
If use rsc compute output applying rsc method'''
assert target.shape[1] == 2
if self.config.RSC.use_rsc:
# making features before avg pooling
features = self.model(input_)
if self.data_parallel:
model1 = self.model.module
else:
model1 = self.model
# do everything after convolutions layers, strating with avg pooling
all_tasks_output = model1.make_logits(features)
logits = (all_tasks_output[0]
if self.config.multi_task_learning
else all_tasks_output)
if isinstance(logits, tuple):
logits = logits[0]
# take a derivative, make tensor, shape as features, but gradients insted features
if self.config.aug.type_aug:
fold_target = target.argmax(dim=1)
target = F.one_hot(fold_target, num_classes=target.shape[1])
target_logits = torch.sum(logits*target, dim=1)
gradients = torch.autograd.grad(target_logits, features,
grad_outputs=torch.ones_like(target_logits),
create_graph=True)[0]
# get value of 1-p quatile
quantile = torch.tensor(np.quantile(a=gradients.data.cpu().numpy(),
q=1-self.config.RSC.p, axis=(1,2,3)),
device=input_.device)
quantile = quantile.reshape(input_.size(0),1,1,1)
# create mask
mask = gradients < quantile
# element wise product of features and mask, correction for expectition value
new_features = (features*mask)/(1-self.config.RSC.p)
# compute new logits
new_logits = model1.spoof_task(new_features)
if isinstance(new_logits, tuple):
new_logits = new_logits[0]
# compute this operation batch wise
random_uniform = torch.rand(size=(input_.size(0), 1), device=input_.device)
random_mask = random_uniform <= self.config.RSC.b
output = torch.where(random_mask, new_logits, logits)
if self.config.loss.loss_type == 'soft_triple':
output = ((output, all_tasks_output[0][1])
if self.config.multi_task_learning
else (output, all_tasks_output[1]))
output = (output, *all_tasks_output[1:])
return output
else:
assert not self.config.RSC.use_rsc
features = self.model(input_)
if self.data_parallel:
model1 = self.model.module
else:
model1 = self.model
output = model1.make_logits(features)
return output
def eval_batch(self, batch_entity_clf: torch.tensor, batch: dict):
batch_size = batch_entity_clf.shape[0]
# get maximum activation (index of predicted entity type)
batch_entity_types = batch_entity_clf.argmax(dim=-1)
# apply entity sample mask
batch_entity_types *= batch['entity_sample_masks'].long()
for i in range(batch_size):
# get model predictions for sample
entity_types = batch_entity_types[i]
# get entities that are not classified as 'None'
valid_entity_indices = entity_types.nonzero().view(-1)
valid_entity_types = entity_types[valid_entity_indices]
valid_entity_spans = batch['entity_spans'][i][valid_entity_indices]
valid_entity_scores = torch.gather(
batch_entity_clf[i][valid_entity_indices], 1,
valid_entity_types.unsqueeze(1)).view(-1)
sample_pred_entities = self._convert_pred_entities(
valid_entity_types, valid_entity_spans, valid_entity_scores)
if not self._overlapping:
sample_pred_entities = self._remove_overlapping(
sample_pred_entities)
self._pred_entities.append(sample_pred_entities)
def eval_batch(self, batch_term_clf: torch.tensor,
batch_rel_clf: torch.tensor, batch_rels: torch.tensor,
batch: dict):
batch_size = batch_rel_clf.shape[0]
rel_class_count = batch_rel_clf.shape[2]
# get maximum activation (index of predicted term type)
batch_term_types = batch_term_clf.argmax(dim=-1)
# apply term sample mask
batch_term_types *= batch['term_sample_masks'].long()
batch_rel_clf = batch_rel_clf.view(batch_size, -1)
# apply threshold to relations
if self._rel_filter_threshold > 0:
batch_rel_clf[batch_rel_clf < self._rel_filter_threshold] = 0
for i in range(batch_size):
# get model predictions for sample
rel_clf = batch_rel_clf[i]
term_types = batch_term_types[i]
# get predicted relation labels and corresponding term pairs
rel_nonzero = rel_clf.nonzero().view(-1)
rel_scores = rel_clf[rel_nonzero]
rel_types = (rel_nonzero % rel_class_count
) + 1 # model does not predict None class (+1)
rel_indices = rel_nonzero // rel_class_count
rels = batch_rels[i][rel_indices]
# get masks of terms in relation
rel_term_spans = batch['term_spans'][i][rels].long()
# get predicted term types
rel_term_types = torch.zeros([rels.shape[0], 2])
if rels.shape[0] != 0:
rel_term_types = torch.stack(
[term_types[rels[j]] for j in range(rels.shape[0])])
# convert predicted relations for evaluation
sample_pred_relations = self._convert_pred_relations(
rel_types, rel_term_spans, rel_term_types, rel_scores)
# get terms that are not classified as 'None'
valid_term_indices = term_types.nonzero().view(-1)
valid_term_types = term_types[valid_term_indices]
valid_term_spans = batch['term_spans'][i][valid_term_indices]
valid_term_scores = torch.gather(
batch_term_clf[i][valid_term_indices], 1,
valid_term_types.unsqueeze(1)).view(-1)
sample_pred_terms = self._convert_pred_terms(
valid_term_types, valid_term_spans, valid_term_scores)
if self._no_overlapping:
sample_pred_terms, sample_pred_relations = self._remove_overlapping(
sample_pred_terms, sample_pred_relations)
self._pred_terms.append(sample_pred_terms)
self._pred_relations.append(sample_pred_relations)
def tensor_to_image(input_tensor: torch.tensor, rgb_map: dict, is_prediction: bool = False) -> \
plt.figure:
"""
Given an input image produces a coloured segmented response for visualization
:param input_tensor: an input image with dims as expected by the nn model
:param rgb_map: rgb_map used in original image
:param is_prediction: a boolean of whether the data is a CXWXH tensor or just 1XWXH
:return: matplotlib figure object
"""
# TODO: the difference in RGB input is an issue here as need to use rgb_map[color] for cityscapes
if is_prediction:
image_tensor = input_tensor.argmax(1)[0]
else:
image_tensor = input_tensor[0]
# create a color palette, selecting a color for each class
colors = []
# TODO: This approach is likely slow, not sure how to improve without passing in dict type
if [type(k) for k in rgb_map.keys()
][0] == int: # if dict is of type int: tuple(list(rgb_triplet)
for color in rgb_map:
colors.append(list(rgb_map[color]))
colors = np.asarray(colors).astype('uint8')
else: # if dict is of type tuple(list(rgb_triplet): int
for color in rgb_map:
colors.append(list(color))
colors = np.asarray(colors).astype('uint8')
# plot the semantic segmentation predictions for each color
r = Image.fromarray(image_tensor.byte().cpu().numpy())
r.putpalette(colors)
fig, ax = plt.subplots()
ax.imshow(r)
return fig
def policy(q_value: tensor, epsilon: float):
# print(q_value.shape)
if [True, False][random.random() < epsilon]:
action = int(q_value.argmax().item())
else:
action = random.sample([0, 1], 1)[0]
return action
def updateStats(self, batch_ne_pred: torch.tensor,
batch_ne_true: torch.tensor, batch_rel_pred: torch.tensor,
batch_rel_true: torch.tensor, sentences_length: List[int]):
batch_size = batch_ne_pred.size(1)
batch_ne_pred = batch_ne_pred.argmax(dim=2)
batch_rel_pred = batch_rel_pred.argmax(dim=3)
for b in range(batch_size):
sl = sentences_length[b]
pred_rel_inds = batch_rel_pred[:sl, :sl,
b].nonzero().detach().cpu().numpy()
true_rel_inds = batch_rel_true[:sl, :sl,
b].nonzero().detach().cpu().numpy()
cur_tp = 0
cur_br = 0
for pr in pred_rel_inds:
for tr in true_rel_inds:
y_pr, x_pr = pr
y_tr, x_tr = tr
pred_rel = batch_rel_pred[y_pr, x_pr, b].item()
pred_e1 = batch_ne_pred[y_pr, b].item()
pred_e2 = batch_ne_pred[x_pr, b].item()
true_rel = batch_rel_true[y_tr, x_tr, b].item()
true_e1 = batch_ne_true[y_tr, b].item()
true_e2 = batch_ne_true[x_tr, b].item()
if (y_pr == y_tr) & (x_pr == x_tr):
cur_br += 1
if (y_pr == y_tr) & (x_pr == x_tr) & (
pred_rel == true_rel) & (pred_e1 == true_e1) & (
pred_e2 == true_e2):
cur_tp += 1
self.TP += cur_tp
self.FP += pred_rel_inds.shape[0] - cur_tp
self.FN += true_rel_inds.shape[0] - cur_tp
self.true_binary_rel += cur_br
self.full_binary_rel += true_rel_inds.shape[0]
def cm(pred: torch.tensor, true: torch.tensor) -> float:
"""
Confusion matrix
"""
c = pred.shape[1]
pred = pred.argmax(dim=1).view(-1)
true = true.view(-1)
mat = torch.zeros(c, c, dtype=torch.long)
mat = mat.index_put_((true, pred), torch.tensor(1), accumulate=True)
return mat.double() / mat.sum()
def update_train_stats(self, loss: torch.tensor, outputs: torch.tensor,
targets: torch.tensor) -> None:
"""[summary]
Arguments:
loss {torch.tensor}
outputs {torch.tensor}
targets {torch.tensor}
"""
acc = (outputs.argmax(1) == targets).sum() / float(targets.shape[0])
self.writer.add_scalar("Loss/train", loss.to(CPU_DEVICE), self.step)
self.writer.add_scalar("Acc/train", acc.to(CPU_DEVICE), self.step)
def acc_satellite(pred: torch.tensor, target: torch.tensor) -> torch.tensor:
"""Calculate accuracy for semantic segmentation
Args:
pred: Predictions
target: Mask with label per pixel
Returns:
One value for the mean accuracy
"""
target = target.squeeze(1)
mask = target != VOID_CODES_RED
mask = target != VOID_CODES_BLACK
return (pred.argmax(dim=1)[mask] == target[mask]).float().mean()
def accuracy(
target: torch.tensor,
output: torch.tensor,
):
""" Computes accuracy
Args:
output: logits of the model
target: true labels
Returns:
accuracy of the predictions
"""
return output.argmax(1).eq(target).double().mean().item()
def select_action(self, value: torch.tensor, eps: float = 0.05):
"""
select actions for DQN
:param value: either action values, shape=(action_dim,), or continuous policy
:param eps: epsilon-greedy threshold
:return: selected action, shape=(1, action_dim); id of selected action, int
"""
flag = random.random()
if flag < 1 - eps: # exploitation, greedily select
a_id = int(value.argmax(dim=1))
else: # exploration, randomly select
a_id = int(torch.rand(1).item() * self.model.num_a)
action = self.model.a_space[a_id, :].reshape((1, -1))
return action, a_id
def updateStats(self, batch_ne_pred: torch.tensor,
batch_ne_true: torch.tensor, sentences_length: List[int]):
batch_size = batch_ne_pred.size(1)
batch_ne_pred = batch_ne_pred.argmax(dim=2)
for b in range(batch_size):
sl = sentences_length[b]
sp = batch_ne_pred[:sl, b].detach().cpu().numpy()
st = batch_ne_true[:sl, b].detach().cpu().numpy()
for i in range(sl):
if (sp[i] == st[i]) & (st[i] == 0):
self.TP += 1
elif (sp[i] != 0) & (st[i] == 0):
self.FP += 1
elif (sp[i] == 0) & (st[i] != 0):
self.FN += 1
def convert_predictions(self,
batch_entity_clf: torch.tensor,
batch_rel_clf: torch.tensor,
batch_rels: torch.tensor,
spans,
span_masks,
rel_filter_threshold: float,
no_overlapping: bool = False):
# get maximum activation (index of predicted entity type)
batch_entity_types = batch_entity_clf.argmax(dim=-1)
# apply entity sample mask
batch_entity_types *= span_masks.long()
# apply threshold to relations
batch_rel_clf[batch_rel_clf < rel_filter_threshold] = 0
batch_pred_entities = []
batch_pred_relations = []
for i in range(batch_rel_clf.shape[0]):
# get model predictions for sample
entity_types = batch_entity_types[i]
entity_spans = spans[i]
entity_clf = batch_entity_clf[i]
rel_clf = batch_rel_clf[i]
rels = batch_rels[i]
# convert predicted entities
sample_pred_entities = self._convert_pred_entities(
entity_types, entity_spans, entity_clf)
# convert predicted relations
sample_pred_relations = self._convert_pred_relations(
rel_clf, rels, entity_types, entity_spans)
# if no_overlapping:
# sample_pred_entities, sample_pred_relations = remove_overlapping(sample_pred_entities,
# sample_pred_relations)
batch_pred_entities.append(sample_pred_entities)
batch_pred_relations.append(sample_pred_relations)
return batch_pred_entities, batch_pred_relations
def eval_batch_entities(self,
batch_entity_clf: torch.tensor,
batch: EvalTensorBatch,
return_conversions=False):
batch_size = batch_entity_clf.shape[0]
# get maximum activation (index of predicted entity type)
batch_entity_types = batch_entity_clf.argmax(dim=-1)
# apply entity sample mask
batch_entity_types *= batch.entity_sample_masks.long()
batch_entities = []
for i in range(batch_size):
# get model predictions for sample
entity_types = batch_entity_types[i]
# get original tokens
tokens = self._sequences[self._evaluated_sequences]
# get entities that are not classified as 'None'
valid_entity_indices = entity_types.nonzero().view(-1)
valid_entity_types = entity_types[valid_entity_indices]
valid_entity_spans = batch.entity_spans[i][valid_entity_indices]
valid_entity_scores = torch.gather(
batch_entity_clf[i][valid_entity_indices], 1,
valid_entity_types.unsqueeze(1)).view(-1)
sample_pred_entities, mapping, json_entities = self._convert_pred_entities(
valid_entity_types, valid_entity_spans, valid_entity_scores)
self._pred_entities.append(sample_pred_entities)
batch_entities.append(sample_pred_entities)
self._predictions_json.append({
"tokens": tokens,
"entities": json_entities,
"relations": [],
"id": hash(" ".join(tokens))
})
self._evaluated_sequences += 1
return batch_entities
def get_accuracy(model_prediction: torch.tensor,
true_prediction: torch.tensor) -> float:
"""
Calculates accuracy with model predictions
Model predictions are the raw model output
True predictions are the labels
:param model_prediction: raw model output
:param true_prediction: labels
:return: accuracy
"""
model_prediction: np.ndarray = model_prediction.detach().cpu().numpy()
true_prediction: np.ndarray = true_prediction.detach().cpu().numpy()
model_prediction = model_prediction.argmax(axis=1)
accuracy_number = np.sum(
model_prediction == true_prediction) / len(true_prediction)
return accuracy_number
def get_f1_score(model_prediction: torch.tensor,
true_prediction: torch.tensor) -> float:
"""
Calculates weighted f1_score with model predictions
Model predictions are the raw model output
True predictions are the labels
:param model_prediction: raw model output
:param true_prediction: labels
:return: f1 score number
"""
model_prediction: np.ndarray = model_prediction.detach().cpu().numpy()
true_prediction: np.ndarray = true_prediction.detach().cpu().numpy()
model_prediction = model_prediction.argmax(axis=1)
f1_score_number = f1_score(y_true=true_prediction,
y_pred=model_prediction,
average='weighted')
return f1_score_number
def default_argmax(
tensor: torch.tensor,
dimension: int) -> torch.LongTensor:
"""Workaround for torch.argmax to enable empty tensors in case the given
dimension is not of size zero.
Returns the indices of the maximum value of all elements in the input
tensor.
Workaround for https://github.com/pytorch/pytorch/issues/28380
Args:
tensor (torch.tensor): the input tensor.
dimension (int): the dimension to reduce.
Returns:
torch.LongTensor:
the indices of the maximum values of a tensor across a dimension.
"""
if tensor.shape[dimension] != 0 and tensor.numel() == 0:
new_shape = list(tensor.shape)
del new_shape[dimension]
return tensor.new_zeros(new_shape)
else:
return tensor.argmax(dimension)
def eval_batch(self,
batch_entity_clf: torch.tensor,
batch_rel_clf: torch.tensor,
batch_rels: torch.tensor,
batch: EvalTensorBatch,
return_conversions=False):
batch_size = batch_rel_clf.shape[0]
rel_class_count = batch_rel_clf.shape[2]
# get maximum activation (index of predicted entity type)
batch_entity_types = batch_entity_clf.argmax(dim=-1)
# apply entity sample mask
batch_entity_types *= batch.entity_sample_masks.long()
batch_rel_clf = batch_rel_clf.view(batch_size, -1)
# apply threshold to relations
if self._rel_filter_threshold > 0:
batch_rel_clf[batch_rel_clf < self._rel_filter_threshold] = 0
batch_entities = []
batch_relations = []
for i in range(batch_size):
# get model predictions for sample
rel_clf = batch_rel_clf[i]
entity_types = batch_entity_types[i]
# get predicted relation labels and corresponding entity pairs
rel_nonzero = rel_clf.nonzero().view(-1)
rel_scores = rel_clf[rel_nonzero]
# rel_types = (rel_nonzero % rel_class_count)
rel_types = (rel_nonzero % rel_class_count
) + 1 # model does not predict None class (+1)
rel_indices = rel_nonzero // rel_class_count
rels = batch_rels[i][rel_indices]
# get masks of entities in relation
rel_entity_spans = batch.entity_spans[i][rels].long()
# get predicted entity types
rel_entity_types = torch.zeros([rels.shape[0], 2])
if rels.shape[0] != 0:
rel_entity_types = torch.stack(
[entity_types[rels[j]] for j in range(rels.shape[0])])
# get original tokens
tokens = self._sequences[self._evaluated_sequences]
# get entities that are not classified as 'None'
valid_entity_indices = entity_types.nonzero().view(-1)
valid_entity_types = entity_types[valid_entity_indices]
valid_entity_spans = batch.entity_spans[i][valid_entity_indices]
valid_entity_scores = torch.gather(
batch_entity_clf[i][valid_entity_indices], 1,
valid_entity_types.unsqueeze(1)).view(-1)
sample_pred_entities, mapping, json_entities = self._convert_pred_entities(
valid_entity_types, valid_entity_spans, valid_entity_scores)
self._pred_entities.append(sample_pred_entities)
# convert predicted relations for evaluation
sample_pred_relations, json_relations = self._convert_pred_relations(
rel_types, rel_entity_spans, rel_entity_types, rel_scores,
mapping)
self._pred_relations.append(sample_pred_relations)
batch_entities.append(sample_pred_entities)
batch_relations.append(sample_pred_relations)
self._predictions_json.append({
"tokens": tokens,
"entities": json_entities,
"relations": json_relations,
"id": hash(" ".join(tokens))
})
self._evaluated_sequences += 1
return batch_entities, batch_relations
def convert_predictions(batch_entity_clf: torch.tensor,
batch_rel_clf: torch.tensor,
batch_rels: torch.tensor,
batch: dict,
rel_filter_threshold: float,
input_reader: BaseInputReader,
no_overlapping: bool = False):
# get maximum activation (index of predicted entity type)
batch_entity_types = batch_entity_clf.argmax(dim=-1)
# apply entity sample mask
batch_entity_types *= batch['entity_sample_masks'].long()
batch_rel_clf_score = batch_rel_clf.clone()
# apply threshold to relations
batch_rel_clf[batch_rel_clf < rel_filter_threshold] = 0
#batch_rel_clf[batch_rel_clf < rel_filter_threshold] = 0
batch_pred_entities = []
batch_pred_entities_scores = []
batch_pred_relations = []
batch_pred_relations_scores = []
for i in range(batch_rel_clf.shape[0]):
# get model predictions for sample
entity_types = batch_entity_types[i]
entity_spans = batch['entity_spans'][i]
entity_clf = batch_entity_clf[i]
rel_clf = batch_rel_clf[i]
rels = batch_rels[i]
rel_clf_score = batch_rel_clf_score[i]
# convert predicted entities
sample_pred_entities = _convert_pred_entities(entity_types,
entity_spans, entity_clf,
input_reader)
valid_entity_indices = entity_types.nonzero().view(-1)
pred_entity_needed_scores = entity_clf[valid_entity_indices]
pred_entity_needed_scores = pred_entity_needed_scores.cpu()
pred_entity_needed_scores = pred_entity_needed_scores.numpy()
#pred_entity_spans_needed = entity_spans[valid_entity_indices]
# convert predicted relations
sample_pred_relations = _convert_pred_relations(
rel_clf, rels, entity_types, entity_spans, input_reader)
rel_nonzero = []
for idx, relation in enumerate(rel_clf):
if relation.nonzero().view(-1).size() != torch.Size([0]):
#print("********,bizare",relation.nonzero().view(-1).size())
rel_nonzero.append(idx)
#rel_nonzero = [idx if torch.nonzero(relation).size()!=0 for idx,relation in enumerate(rel_clf))]
#rel_nonzero = rel_clf.nonzero().view(-1)
pred_rel_scores = rel_clf_score[rel_nonzero] #######ici
pred_rel_scores = pred_rel_scores.cpu()
pred_rel_scores = pred_rel_scores.numpy()
if no_overlapping:
sample_pred_entities, sample_pred_relations = remove_overlapping(
sample_pred_entities, sample_pred_relations)
batch_pred_entities_scores.append(pred_entity_needed_scores)
batch_pred_relations_scores.append(pred_rel_scores)
batch_pred_entities.append(sample_pred_entities)
batch_pred_relations.append(sample_pred_relations)
#print("*****1",len(batch_pred_entities_scores))
#print("*****2",len(batch_pred_entities))
return batch_pred_entities, batch_pred_relations, batch_pred_entities_scores, batch_pred_relations_scores
def __call__(self, outputs: torch.tensor,
scalars: typing.Optional[torch.tensor] = None) -> torch.tensor:
if scalars is None:
return outputs.argmax(dim=1).float()
scalars[:] = outputs.argmax(dim=1).float()
return scalars