def save(self, dir_path, file_type):
"""
Saves the MacroF1's state to file.
:param dir_path : The path to the dir in which to save the Loss' state.
:param file_type: The type of the file to be saved. That is, a regular checkpoint, '.ckp', or the best state,
'.bst', so far.
:return: Nothing
"""
# Get base state dict, common to all metrics.
state = super()._save_state()
# Extend the base state dict with the extra information relevant for the Loss metric.
state["seen_classes_to_idx"] = self._seen_classes_to_idx
state["unseen_classes_to_idx"] = self._unseen_classes_to_idx
state['print_info_max_length'] = self._print_info_max_length
torch.save(state, join_path([dir_path] + [self._name + file_type]))
# Now we save each sub-metric. First we guarantee that the correct dir exists.
sub_metrics_dir_path = join_path([dir_path] + [self._name + '_sub_metrics' + os.sep])
directory = os.path.dirname(sub_metrics_dir_path)
if (not os.path.exists(directory)):
os.makedirs(directory)
for _, mf1 in self.seen_unseen_MacroF1.items():
mf1.save(sub_metrics_dir_path, file_type)
def update_classes_info(self,
classes_in_split_to_idx_map=None,
is_seen_class_indicator=None):
"""
Sets the parameters used to determine if early stopping should occur.
:param classes_in_split_to_idx_map: A dictionary that maps classes names to their corresponding index, as
defined in the dataset being used.
:param is_seen_class_indicator : A dictionary that indicates whether a class belongs to the 'seen' set or
the 'unseen' set.
:return: Nothing
"""
# Get the correct load_path for the sub-F1 metrics.
if (self._optional_seen_unseen != ''):
load_path = None if self._load_path is None else join_path(
get_path_components(self._load_path)[:-1])
load_path = None if self._load_path is None else join_path(
[load_path] + [
self._optional_seen_unseen + '_' + self._name +
'_sub_metrics' + os.sep
])
else:
load_path = None if self._load_path is None else join_path(
[self._load_path] + [self._name + '_sub_metrics' + os.sep])
class_overlap = False
for new_class in classes_in_split_to_idx_map:
if (new_class not in self._classes_in_split_to_idx_map):
class_file_name = re.sub('/', '-', re.sub(' ', '_', new_class))
class_load_path = None if self._load_path is None else join_path(
[load_path] + [class_file_name + '_'])
self._classes_in_split_to_idx_map[
new_class] = classes_in_split_to_idx_map[new_class]
self._classes_individual_F1_scores[new_class] = F1(
self._evaluated_on_batches, self._max_num_batches,
self._accumulate_n_batch_grads, self._training_batch_sizes,
class_load_path, self._load_epoch_or_best,
len(self._batch_history_x) + 1, new_class)
# self._is_seen_class_indicator[new_class] = is_seen_class_indicator[new_class]
else:
class_overlap = True
self._idx_to_classes_in_split_map = [
_class for _class, idx in sorted(
self._classes_in_split_to_idx_map.items(), key=lambda x: x[1])
]
if (class_overlap):
print_warning(
"When adding new classes to the 'MacroF1' metric, some of these new classes were already "
+ "being tracked. The new setting WILL BE IGNORED.")
def __init__(self, path_to_embeddings, masking_type='sub_obj_masking', dataset_type='DEBUG', setting='N', fold='0',
ulbs_type='basic'):
"""Docstring.TODO"""
path = join_path([_data_paths.UW_RE_UVA_PROPOSED_SPLITS, masking_type, dataset_type, setting, fold])
with open(join_path([path, "rel_descs_" + ulbs_type + ".ulbs"]), 'rb') as f:
self._idxs_of_rels_descs_in_train = pickle.load(f)
self._ulbs_of_rels_descs_in_train = pickle.load(f)
self._path_to_embeddings = path_to_embeddings
def get_UW_RE_UVA_datasets(masking_type='sub_obj_masking', dataset_type='DEBUG', setting='N', fold='0', validation=True,
batch_size_xs=25, batch_size_xs_eval=10, num_workers_xs=0, eval_all_descriptions=True,
labels_to_load=['supervised_lbls'], dataset_debug=None, **dataset_arguments):
path_setting = setting.split('-')[0] if setting in ['ZS-O', 'ZS-C', 'GZS-O', 'GZS-C'] else setting
path = join_path([_data_paths.UW_RE_UVA_PROPOSED_SPLITS, masking_type, dataset_type, path_setting, fold])
train_dataset = UW_RE_UVA(join_path([path, "train"]), setting, batch_size_xs, labels_to_load, num_workers_xs,
eval_all_descriptions, dataset_debug)
if (validation):
val_dataset = UW_RE_UVA(join_path([path, "val"]), setting, batch_size_xs_eval, labels_to_load, num_workers_xs,
eval_all_descriptions, dataset_debug)
else:
val_dataset = None
test_dataset = UW_RE_UVA(join_path([path, "test"]), setting, batch_size_xs_eval, labels_to_load, num_workers_xs,
eval_all_descriptions, dataset_debug)
return train_dataset, val_dataset, test_dataset
def __init__(self, masking_type='sub_obj_masking', dataset_type='DEBUG'):
"""Docstring.TODO"""
super(UW_RE_UVA_EMBEDDING_LAYER_PRE_TRAIN, self).__init__()
self._data_path = join_path([_data_paths.UW_RE_UVA_PROPOSED_SPLITS, masking_type,
("DEBUG_" if dataset_type == 'DEBUG' else "") + "sentences_to_ELMo.hdf5"])
data = h5py.File(self._data_path, 'r')
self._num_instances = len(data) - 1
data.close()
def __init__(self, evaluated_on_batches, max_num_batches, accumulate_n_batch_grads, training_batch_sizes,
load_path, load_epoch_or_best, starting_batch_num=1):
"""
Initiates a HarmonicMacroF1 metric object.
:param evaluated_on_batches: Determines whether this specific instance of a HarmonicMacroF1 metric is evaluated
for minibatches. Used for plotting.
:param max_num_batches : The maximum number of batches in an epoch. Used to aggregate epoch information.
:param load_path : The path that leads to a saved state of an instance of the HarmonicMacroF1 metric.
:param load_epoch_or_best : This parameter identifies whether to load the state associated with a specific
epoch or the state of the epoch in which the model performed the best.
:param starting_batch_num : Some metrics can be added to the MetricsManager after the model has already been
trained for a few epochs. This identifies the first batch in which this particular
instance was initiated.
"""
super().__init__("HMF1", evaluated_on_batches, max_num_batches, accumulate_n_batch_grads, training_batch_sizes,
False, load_path, load_epoch_or_best, starting_batch_num)
# Variables specific to the HarmonicMacroF1 metric.
load_path_seen = None if load_path is None else (join_path([load_path] + [self._name + '_sub_metrics'] + ['seen']))
load_path_unseen = None if load_path is None else (join_path([load_path] + [self._name + '_sub_metrics'] + ['unseen']))
self._seen_unseen_MacroF1 = {'seen': MacroF1(evaluated_on_batches, max_num_batches, accumulate_n_batch_grads,
training_batch_sizes, load_path_seen, load_epoch_or_best,
starting_batch_num, 'seen'),
'unseen': MacroF1(evaluated_on_batches, max_num_batches, accumulate_n_batch_grads,
training_batch_sizes, load_path_unseen, load_epoch_or_best,
starting_batch_num, 'unseen'),
}
self._seen_classes_to_idx = None if self._metric_state is None else self._metric_state['seen_classes_to_idx']
self._unseen_classes_to_idx = None if self._metric_state is None else self._metric_state['unseen_classes_to_idx']
# Variables used to determine the length of the information to be printed.
self._print_info_max_length = 14 if self._metric_state is None else self._metric_state['print_info_max_length']
def save(self, dir_path, file_type):
"""
Saves the metric's state to file.
:param dir_path : The path to the dir in which to save the metric's state.
:param file_type: The type of the file to be saved. That is, a regular checkpoint, '.ckp', or the best state,
'.bst', so far.
:return: Nothing
"""
torch.save(self._save_state(),
join_path([dir_path] + [self._name + file_type]))
def save(self, dir_path, file_type):
"""
Saves the F1's state to file.
:param dir_path : The path to the dir in which to save the F1's state.
:param file_type: The type of the file to be saved. That is, a regular checkpoint, '.ckp', or the best state,
'.bst', so far.
:return: Nothing
"""
# Get base state dict, common to all metrics.
state = super()._save_state()
# Extend the base state dict with the extra information relevant for the Loss metric.
state["optional_class_name"] = self._optional_class_name
state["precision_batch_x"] = self._precision_batch_x
state["precision_batch"] = self._precision_batch
state["precision_epoch_x"] = self._precision_epoch_x
state["precision_epoch"] = self._precision_epoch
state["recall_batch_x"] = self._recall_batch_x
state["recall_batch"] = self._recall_batch
state["recall_epoch_x"] = self._recall_epoch_x
state["recall_epoch"] = self._recall_epoch
optional_class_name_part = re.sub(
'/', '-', re.sub(' ', '_', self._optional_class_name))
optional_class_name_part += ('_' if self._optional_class_name != ''
else '')
torch.save(
state,
join_path([dir_path] +
[optional_class_name_part + self._name + file_type]))
def __init__(self, short_name, evaluated_on_batches, max_num_batches,
accumulate_n_batch_grads, training_batch_sizes,
lower_is_better, load_path, load_epoch_or_best,
starting_batch_num):
"""
Instantiates a Metric (or child of Metric) object.TODO: Missing parameters
:param short_name : The short name by which the metric is identified.
:param evaluated_on_batches: Determines whether the metric is evaluated for minibatches. Used for plotting.
:param max_num_batches : The maximum number of batches in an epoch. Used to aggregate epoch information. See
specific uses in child classes.
:param lower_is_better : Identifies whether a lower value of the metric indicates that the model performs
better.
:param load_path : The path that leads to a saved state of an instance of this metric.
:param load_epoch_or_best : This parameter identifies whether to load the state associated with a specific
epoch or the state of the epoch in which the model performed the best.
:param starting_batch_num : Some metrics can be added to the MetricsManager after the model has already been
trained for a few epochs. This parameter identifies at which batch num the metric
was started.
"""
# Names.
self._name = self.__class__.__name__
self._short_name = short_name
# Load previously saved metric state.
self._load_path = load_path
self._load_epoch_or_best = load_epoch_or_best
extra_name = ''
if (load_path is not None):
if (load_path[-1] != os.sep):
path_components = get_path_components(load_path)
extra_name = path_components[-1]
load_path = join_path(path_components[:-1]) + os.sep
self._metric_state = trainer_helpers.load_checkpoint_state(
load_path, extra_name + self._name, load_epoch_or_best)
# Evaluation history.
self._batch_history = [] if self._metric_state is None else self._metric_state[
'batch_history']
self._batch_history_x = [] if self._metric_state is None else self._metric_state[
'batch_history_x']
self._epoch_history = [] if self._metric_state is None else self._metric_state[
'epoch_history']
self._epoch_history_x = [] if self._metric_state is None else self._metric_state[
'epoch_history_x']
# Used to (only) evaluate the model's performance, so that no state of the singular evaluation is kept.
self._eval_only_history = None
# Batch information.
self._evaluated_on_batches = evaluated_on_batches
self._max_num_batches = max_num_batches
self._accumulate_n_batch_grads = accumulate_n_batch_grads
self._training_batch_sizes = training_batch_sizes
if (training_batch_sizes[0] is None):
self._last_acc_batch_start_num = None
self._normal_acc_batch_size = None
self._last_acc_batch_size = None
else:
max_b = max_num_batches
acc = accumulate_n_batch_grads
self._last_acc_batch_start_num = max_b - max_b % acc + (
1 if max_b % acc >= 1 else -acc + 1)
self._normal_acc_batch_size = acc * training_batch_sizes[0]
self._last_acc_batch_size = (max_b - self._last_acc_batch_start_num
) * training_batch_sizes[0]
self._last_acc_batch_size += training_batch_sizes[1]
self._starting_batch_num = starting_batch_num #TODO: Should load from memory for existing metrics
# Early stopping and save best parameters
self._lower_is_better = lower_is_better
self._early_stop_tolerance = None
self._patience = None
# Variables used to determine the length of the information to be printed.
self._print_info_max_length = 0
# Class information parameters. Used in metrics such as F1, MacroF1 and HarmonicMacroF1.
self._classes_in_split_to_idx_map = {}
self._idx_to_classes_in_split_map = []
self._is_seen_class_indicator = {}
def __init__(self, file_name, setting, batch_size_xs, labels_to_load, num_workers_xs=0,
eval_all_descriptions=True, dataset_debug=None, full_prob_interp=False):
"""
Instantiates a UW_RE_UVA dataset object for the specific split associated with 'file_name'.
TODO: Missing variables.
:param file_name : The (almost complete) path to the data, in storage, pertaining this dataset.
:param batch_size_xs : The number of data instances (sentences x) (and respective labels) to load.
:param labels_to_load : A list that indicates what kind of labels are meant to be loaded by this dataset.
:param num_workers_xs : The number of sub-processes loading data instances from storage.
:param eval_all_descriptions: Determines whether to evaluate against all relation descriptions or not.
"""
# Input variables
self._file_name = file_name
self._batch_size_xs = batch_size_xs
self._labels_to_load = labels_to_load
self._num_workers_xs = num_workers_xs
self._eval_all_descriptions = eval_all_descriptions
# TODO: Always load the supervised labels, as they are necessary for evaluation in any of the splits.
# TODO: Instead change the labels_to_load to refer uniquely to which unsupervised labels are meant to be loaded.
# Instantiate the sub-dataset that will load the data instances and the corresponding labels.
self._x_sentences_dataset = UW_RE_UVA_xs(file_name, setting, labels_to_load, dataset_debug, full_prob_interp)
# Determine if this pertains a train split. If so we sample one relation description per relation in the split.
self._train_split = self._x_sentences_dataset.train_split
if (self._batch_size_xs == -1 or self._batch_size_xs > self._x_sentences_dataset.num_instances):
if (self._batch_size_xs > self._x_sentences_dataset.num_instances):
print_warning("Split: " + self._x_sentences_dataset.split + " | Requested Batch Size is bigger than " +
"available number of instances. Setting Batch Size to be equal to the available " +
"number of instances.")
self._batch_size_xs = self._x_sentences_dataset.num_instances
self._x_sentences_dataloader = DataLoader(self._x_sentences_dataset, batch_size=self._batch_size_xs,
shuffle=self._train_split, num_workers=num_workers_xs,
collate_fn=PadCollate(self._x_sentences_dataset.valid_labels))
self._x_sentences_dataloader_iter = iter(self._x_sentences_dataloader)
# We create a generator that allows us to sample from the x_sentences_dataset indefinitely.
def get_subdataset_batch(dl):
while (True):
for batch in dl:
yield batch
self._infinite_x_sentences_dl = get_subdataset_batch(self._x_sentences_dataloader)
# Determine the correct length of this meta-dataset
num_full_x_batches = len(self._x_sentences_dataset) // self._batch_size_xs
equal_size_last_x_batch = len(self._x_sentences_dataset) % self._batch_size_xs == 0
self._len_this_meta_dataset = num_full_x_batches + (0 if equal_size_last_x_batch else 1)
# Load the list of relations involved in this split (in the file there is one relation per line)
self._relations_in_split = self._x_sentences_dataset.relations_in_split
# Load the map between relations and the indices of relation_descriptions
path_components = get_path_components(file_name)
with open(join_path(path_components[:-6] + ["ELMO_rel_descs_idxs.map"]), 'rb') as f:
self._relation_to_idxs_rel_description = pickle.load(f)
# The map above conveys always the same information for a specific instance of a dataset split, so:
self._descs_idxs = [self._relation_to_idxs_rel_description[rel] for rel in self._relations_in_split]
# Remove duplicate indices (for relation descriptions that might describe more than one relation).
self._descs_idxs_all = []
for idxs_set in self._descs_idxs:
for idx in idxs_set:
if (idx not in self._descs_idxs_all):
self._descs_idxs_all.append(idx)
# Indicates which indices are meant to be aggregated together, when comparing against multiple relation
# descriptions for the same relation.
self._aggregate = []
for idxs_set in self._descs_idxs:
map_to_output_idxs = []
for idx in idxs_set:
map_to_output_idxs.append(self._descs_idxs_all.index(idx))
self._aggregate.append(torch.LongTensor(map_to_output_idxs))
def __init__(self, file_name, setting, labels_to_load, dataset_debug=None, full_prob_interp=False):
"""
Instantiates a UW_RE_UVA_xs dataset object for the specific split associated with 'file_name'.
:param file_name : The (almost complete) path to the data, in storage, pertaining this dataset.
:param labels_to_load: A list that indicates what kind of labels are meant to be loaded by this dataset.
"""
self._file_name = file_name
self._labels_to_load = labels_to_load
self._full_prob_interp = full_prob_interp
path_components = get_path_components(file_name)
debug_file = "DEBUG_" if path_components[-4] == 'DEBUG' else ""
self._sentences_hdf5_file = join_path(path_components[:-4] + [debug_file + "sentences_to_ELMo.hdf5"])
self._setting = setting
self._split = path_components[-1]
# Catch undesired configurations from the start.
if (self._setting != 'N' and self._setting[0] != 'G' and full_prob_interp):
raise RuntimeError("A full probabilistic interpretation is desired, but the chosen setting (" +
self._setting + ') is incompatible with such specification. Compatible settings are: ' +
"Normal (N); Generalised Any-Shot settings (GZS-(O/C), GFS-1, GFS-2, GFS-5, GFS-10).")
# print('\n\n\n')
DEBUG_EXPERIMENT = dataset_debug
# print('DEBUG_EXPERIMENT:', dataset_debug)
if (DEBUG_EXPERIMENT is not None and self._split == 'test'):
self._file_name = join_path(get_path_components(self._file_name)[:-1] + ['val'])
# print('FILE NAME:', self._file_name)
# Get the instances indices (on the main hdf5 file) and determine the number of instances in this split.
with open(self._file_name + ".idxs", 'rb') as f:
loaded_instances = pickle.load(f)
if (self._split != 'train'):
# print(self._split, ' ORIGINAL INSTANCE INDICES', loaded_instances)
if (DEBUG_EXPERIMENT == 'instances'):
self._instances_indices = loaded_instances[(0 if self._split == 'val' else 1)::2]
elif (DEBUG_EXPERIMENT == 'classes'):
seen_classes = set()
seen_instances = []
seen_labels = []
if (self._setting[0] == 'G'):
with open(join_path(path_components[:-1] + ["train_relations.txt"]), 'r',
encoding='utf-8') as f:
for line in f:
seen_classes.add(line.strip())
with open(self._file_name + '.lbs', 'rb') as f:
temp_labels = pickle.load(f)
count_class_elements = {}
for inst_num, instance in enumerate(loaded_instances):
if (temp_labels[inst_num] not in seen_classes):
if (temp_labels[inst_num] not in count_class_elements):
count_class_elements[temp_labels[inst_num]] = [instance]
else:
count_class_elements[temp_labels[inst_num]].append(instance)
else:
seen_instances.append(instance)
seen_labels.append(temp_labels[inst_num])
sorted_by_class_num_insts = sorted(count_class_elements.items(), key=lambda kv: len(kv[1]), reverse=True)
classes_and_instances = sorted_by_class_num_insts[(1 if self._split == 'val' else 0)::2]
self._labels = [_class for _class_set, inst_nums in classes_and_instances
for _class in [_class_set] * len(inst_nums)] + seen_labels
self._instances_indices = [inst_num for _, inst_nums in classes_and_instances
for inst_num in inst_nums] + seen_instances
elif (DEBUG_EXPERIMENT is None):
self._instances_indices = loaded_instances
else:
raise ValueError('Wrong DEBUG_EXPERIMENT value.')
else:
self._instances_indices = loaded_instances
self._num_instances = len(self._instances_indices)
self._valid_labels = []
# Load the corresponding supervised labels (text based label, i.e. the actual relation name).
if ('supervised_lbls' in labels_to_load):
with open(self._file_name + '.lbs', 'rb') as f:
loaded_labels = pickle.load(f)
if (self._split != 'train'):
# print(self._split, ' ORIGINAL LABELS', loaded_labels)
if (DEBUG_EXPERIMENT == 'instances'):
self._labels = loaded_labels[(0 if self._split == 'val' else 1)::2]
elif (DEBUG_EXPERIMENT == 'classes'):
pass
elif (DEBUG_EXPERIMENT is None):
self._labels = loaded_labels
else:
raise ValueError('Wrong DEBUG_EXPERIMENT value.')
else:
self._labels = loaded_labels
self._valid_labels.append('supervised_lbls')
# TODO: remove this when done debugging weird results on test set.
set_labels = set(self._labels)
# Load the list of classes involved in this split. For some settings (like Few-Shot settings or Generalised
# Any-Shot settings) the classes of another split might be present in this specific split.
self._relations_in_split = []
self._relation_in_split_to_idx_map = {}
self._is_seen_class_indicator = {}
self._exclusive_split_relations = []
with open(self._file_name + "_relations.txt", 'r', encoding='utf-8') as f:
r_num = 0
for line in f:
# for r_num, line in enumerate(f):
relation = line.strip()
if (relation in set_labels):
self._exclusive_split_relations.append(relation)
self._relations_in_split.append(relation)
self._relation_in_split_to_idx_map[relation] = r_num
r_num += 1
if (self._setting[0] == 'G' or self._setting[0] == 'F' or self._setting == 'ZS-C'):
# If this is a Generalised setting and a validation or test split, get the train classes.
if (self._setting[0] == 'G' and self._split != 'train'):
for relation in self._exclusive_split_relations:
self._is_seen_class_indicator[relation] = False
self._relations_from_another_split = []
with open(join_path(path_components[:-1] + ["train_relations.txt"]), 'r', encoding='utf-8') as f:
for r_num, line in enumerate(f):
relation = line.strip()
self._relations_from_another_split.append(relation)
self._relations_in_split.append(relation)
self._relation_in_split_to_idx_map[relation] = r_num + len(self._exclusive_split_relations)
self._is_seen_class_indicator[relation] = True
# If this is a Few-Shot setting and a train split, get the validation and test classes.
# Also, if this is a (Generalised) Zero-Shot Closed setting, get the validation and test classes.
if ((self._setting[0] == 'F' or self._setting[1] == 'F' or
self._setting == 'ZS-C' or self._setting == 'GZS-C') and self._split == 'train'):
for relation in self._exclusive_split_relations:
self._is_seen_class_indicator[relation] = True
# TODO: This assumes there's a validation split, which might not be the case.
self._relations_from_another_split = []
for split in (['val', 'test'] if DEBUG_EXPERIMENT is None else ['val']):
with open(join_path(path_components[:-1] + [split + "_relations.txt"]), 'r', encoding='utf-8') as f:
for r_num, line in enumerate(f):
relation = line.strip()
self._relation_in_split_to_idx_map[relation] = len(self._relations_from_another_split) + len(self._exclusive_split_relations)
self._relations_from_another_split.append(relation)
self._relations_in_split.append(relation)
self._is_seen_class_indicator[relation] = False
# If this pertains a train split, load the unsupervised sentence labels, if they have been requested.
# TODO: maybe some of these checks can be done when reading labels_to_load with argparse.
if ('u_sentence_lbls' in labels_to_load and self._split == 'train'):
with open(self._file_name + '.ulbs', 'rb') as f:
self._data_as_idxs = pickle.load(f)
self._valid_labels.append('u_sentence_lbls')
##### IMPORTS #####
###################
# *** General modules imports. *** #
# Import here!
# *** Import own functions. *** #
from helpers.general_helpers import join_path
######################
##### DATA PATHS #####
######################
DATASETS = join_path(["datasets", "Data"])
# *** MNIST *** #
MNIST = join_path([DATASETS, "MNIST"]) # Raw MNIST data
# *** Levy *** #
LEVY = join_path([DATASETS, "Levy"])
LEVY_FULL_POSITIVE = join_path([LEVY, "positive_examples"])
# *** Ours *** #
UW_RE_UVA = join_path([DATASETS, "Ours"])
UW_RE_UVA_RELATIONS_NAMES = join_path([UW_RE_UVA, "relation_names.txt"])
UW_RE_UVA_RELATIONS_DESCRIPTIONS = join_path(
[UW_RE_UVA, "relations_descriptions"])
UW_RE_UVA_PROPOSED_SPLITS = join_path([UW_RE_UVA, "proposed_splits"])
UW_RE_UVA_PROPOSED_SPLITS_NEW = join_path([UW_RE_UVA, "proposed_splits_new"])
