def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
input_defs = {
self.key_targets:
DataDefinition(
[-1], [torch.Tensor],
"Batch of targets, each being a single index [BATCH_SIZE]")
}
if self.use_prediction_distributions:
input_defs[self.key_predictions] = DataDefinition([-1, -1], [
torch.Tensor
], "Batch of predictions, represented as tensor with probability distribution over classes [BATCH_SIZE x NUM_CLASSES]"
)
else:
input_defs[self.key_predictions] = DataDefinition([-1], [
torch.Tensor
], "Batch of predictions, represented as tensor with indices of predicted answers [BATCH_SIZE]"
)
if self.use_masking:
input_defs[self.key_masks] = DataDefinition(
[-1], [torch.Tensor], "Batch of masks [BATCH_SIZE]")
return input_defs
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
if self.mode_reverse:
if self.use_input_distributions:
return {
self.key_inputs:
DataDefinition([-1, -1, -1], [
torch.Tensor
], "Batch of sentences represented as a single tensor with batch of probability distributions [BATCH_SIZE x SEQ_LENGTH x ITEM_SIZE]"
),
}
else:
return {
self.key_inputs:
DataDefinition([-1, -1], [
torch.Tensor
], "Batch of sentences represented as a single tensor of indices of particular words [BATCH_SIZE x SEQ_LENGTH]"
),
}
else:
return {
self.key_inputs:
DataDefinition([-1, -1, 1], [
list, list, str
], "Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]"
),
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
if self.return_feature_maps:
return {
self.key_outputs:
DataDefinition([
-1, self.feature_maps_depth, self.feature_maps_height,
self.feature_maps_width
], [
torch.Tensor
], "Batch of feature maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]"
)
}
else:
return {
self.key_outputs:
DataDefinition([-1, self.output_size], [
torch.Tensor
], "Batch of outputs, each represented as probability distribution over classes [BATCH_SIZE x PREDICTION_SIZE]"
)
}
def input_data_definitions(self):
return {
"input1": DataDefinition([-1, 1], [list, int], "comment1"),
"input2": DataDefinition([-1, -1, -1], [list, list, str],
"comment2"),
"input3": DataDefinition([-1, -1], [float], "comment3")
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
# Add targets.
input_defs = {
self.key_targets:
DataDefinition([-1, -1], [
torch.Tensor
], "Batch of sentences represented as a single tensor of indices of particular words [BATCH_SIZE x SEQ_LENGTH]"
),
}
# Add predictions.
if self.use_prediction_distributions:
input_defs[self.key_predictions] = DataDefinition([-1, -1, -1], [
torch.Tensor
], "Batch of predictions, represented as tensor with sequences of probability distributions over classes [BATCH_SIZE x SEQ_LENGTH x NUM_CLASSES]"
)
else:
input_defs[self.key_predictions] = DataDefinition([-1, -1], [
torch.Tensor
], "Batch of predictions, represented as tensor with sequences of indices of predicted answers [BATCH_SIZE x SEQ_LENGTH]"
)
# Add masks.
#if self.use_masking:
# input_defs[self.key_masks] = DataDefinition([-1, -1], [torch.Tensor], "Batch of masks (separate mask for each sequence in the batch) [BATCH_SIZE x SEQ_LENGTH]")
return input_defs
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_indices:
DataDefinition([-1, 1], [list, int],
"Batch of sample indices [BATCH_SIZE] x [1]"),
self.key_inputs:
DataDefinition([-1, 1, self.height, self.width], [
torch.Tensor
], "Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]"
),
self.key_targets:
DataDefinition(
[-1], [torch.Tensor],
"Batch of targets, each being a single index [BATCH_SIZE]"),
self.key_labels:
DataDefinition([-1, 1], [
list, str
], "Batch of targets, each being a single word [BATCH_SIZE] x [STRING]"
)
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).
"""
return {
self.key_indices:
DataDefinition([-1, 1], [list, int],
"Batch of sample indices [BATCH_SIZE] x [1]"),
self.key_images:
DataDefinition([-1, -1, -1, -1], [
torch.Tensor
], "Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]"
),
self.key_labels:
DataDefinition([-1, 1], [
list, str
], "Batch of target labels, each being a single word [BATCH_SIZE] x [STRING]"
),
self.key_answers:
DataDefinition([-1, 1], [
list, str
], "Batch of predicted labels, each being a single word [BATCH_SIZE] x [STRING]"
)
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
d = {}
if self.prediction_mode == "Dense":
d[self.key_predictions] = DataDefinition([
-1, -1, self.prediction_size
], [
torch.Tensor
], "Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]"
)
elif self.prediction_mode == "Last": # "Last"
# Only last prediction.
d[self.key_predictions] = DataDefinition([
-1, self.prediction_size
], [
torch.Tensor
], "Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]"
)
# Output hidden state stream TODO: why do we need that?
if self.output_last_state:
d[self.key_output_state] = DataDefinition([
-1, 1, self.hidden_size
], [
torch.Tensor
], "Batch of RNN final hidden states [BATCH_SIZE x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]"
)
return d
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_feature_maps: DataDefinition([-1, self.feature_maps_depth, self.feature_maps_height, self.feature_maps_width], [torch.Tensor], "Batch of feature maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]"),
self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], "Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]"),
}
def output_data_definitions(self):
"""
Function returns a empty dictionary with definitions of output data produced the component.
:return: Empty dictionary.
"""
return {
self.key_masks: DataDefinition([-1], [torch.Tensor], "Batch of masks [BATCH_SIZE]"),
self.key_string_indices: DataDefinition([-1], [torch.Tensor], "Batch of indices corresponging to inputs strings when using provided word mappings [BATCH_SIZE]")
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_image_encodings: DataDefinition([-1, self.image_encoding_size], [torch.Tensor], "Batch of encoded images [BATCH_SIZE x IMAGE_ENCODING_SIZE]"),
self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], "Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]"),
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_targets: DataDefinition([-1]*self.num_targets_dims, [torch.Tensor], "Batch of targets (indices) [DIM 1 x DIM 2 x ... ]"),
self.key_predictions: DataDefinition([-1]*(self.num_targets_dims+1), [torch.Tensor], "Batch of predictions, represented as tensor with probability distribution over classes [DIM 1 x DIM x ... x NUM_CLASSES]")
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
if self.mode_detokenize == False:
return { self.key_outputs: DataDefinition([-1, -1, 1], [list, list, str], "Batch of tokenized sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]") }
else:
return { self.key_outputs: DataDefinition([-1, 1], [list, str], "Batch of sentences, each represented as a single string [BATCH_SIZE] x [string]") }
def test_override(self):
""" Tests whether values cannot be overriden. """
dd = DataDefinition([1], [int], "Value")
with self.assertRaises(AttributeError):
dd.dimensions = [1, 2]
with self.assertRaises(AttributeError):
dd.types = [str, list]
with self.assertRaises(AttributeError):
dd.description = "New Description"
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"),
self.key_sources: DataDefinition([-1, self.sentence_length, 1], [list, list, str], "Batch of input sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]"),
self.key_targets: DataDefinition([-1, self.sentence_length, 1], [list, list, str], "Batch of target sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]")
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"),
#self.key_inputs: DataDefinition([-1, self.context, 1], [list, list, str], "Batch of sentences, each being a context consisint of several words [BATCH_SIZE] x [CONTEXT_SIZE] x [WORD]"),
self.key_inputs: DataDefinition([-1, 1], [list, str], "Batch of sentences, each being a context consisint of several words [BATCH_SIZE] x [string CONTEXT_SIZE * WORD]"),
self.key_targets: DataDefinition([-1, 1], [list, str], "Batch of targets, each being a single word [BATCH_SIZE] x [WORD]")
}
def test_values(self):
""" Tests whether the values are set. """
dd = DataDefinition([1], [int], "Value")
self.assertEqual(dd.dimensions, [1])
self.assertEqual(dd.types, [int])
self.assertEqual(dd.description, "Value")
def output_data_definitions(self):
"""
Function returns a empty dictionary with definitions of output data produced the component.
:return: Empty dictionary.
"""
return {
self.key_output_indices:
DataDefinition([-1], [torch.Tensor],
"Batch of merged (output) indices [BATCH_SIZE]"),
self.key_output_strings:
DataDefinition([-1, 1], [
list, str
], "Batch of merged strings, corresponging to indices when using the provided word mappings [BATCH_SIZE] x [STRING]"
)
}
def test_extension_definitions(self):
""" Tests extension of output definition keys. """
all_defs = {}
# Key not existing in output definitions - ADD.
all_defs["output2"] = DataDefinition([-1, -1, -1], [list, list, str],
"comment")
self.assertEqual(
self.component.export_output_definitions(all_defs,
log_errors=False), 0)
# Key already existing in output definitions.
all_defs["output"] = DataDefinition([-1, -1, -1], [list, list, str],
"comment")
self.assertEqual(
self.component.export_output_definitions(all_defs,
log_errors=False), 1)
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_inputs: DataDefinition([-1, -1, 1], [list, list, str], "Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]")
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_outputs: DataDefinition([-1, -1, self.embeddings_size], [torch.Tensor], "Batch of embedded sentences [BATCH_SIZE x SENTENCE_LENGTH x EMBEDDING_SIZE]")
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_inputs: DataDefinition([-1, self.input_size], [torch.Tensor], "Batch of inputs, each represented as index [BATCH_SIZE x INPUT_SIZE]"),
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of indices [BATCH_SIZE] x [1]")
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_predictions: DataDefinition([-1, self.prediction_size], [torch.Tensor], "Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x PREDICTION_SIZE]")
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], "Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]")
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).
"""
return {
self.key_strings: DataDefinition([-1, 1], [list, str], "Batch of strings, each being treated as a single 'vocabulary entry' (word) [BATCH_SIZE] x [STRING]")
}
def test_handshake_input_definitions_types(self):
""" Tests handskaking of input definition keys. """
all_defs = {}
all_defs["input2"] = DataDefinition([-1, -1, -1], [list, list, str],
"comment2")
all_defs["input3"] = DataDefinition([-1, -1], [float], "comment3")
# One input with wrong number of types.
all_defs["input1"] = DataDefinition([-1, 1], [list], "comment")
self.assertEqual(
self.component.handshake_input_definitions(all_defs,
log_errors=False), 1)
# One input with one wrong type.
all_defs["input1"] = DataDefinition([-1, 1], [list, str], "comment")
self.assertEqual(
self.component.handshake_input_definitions(all_defs,
log_errors=False), 1)
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_outputs: DataDefinition([-1], [torch.Tensor], "Batch of labels, each represented as a single index [BATCH_SIZE]")
}
def input_data_definitions(self):
"""
Function returns a dictionary with definitions of input data that are required by the component.
:return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_inputs: DataDefinition([-1, -1, self.bow_size], [list, list, torch.Tensor], "Batch of sentences, each represented as a list of vectors [BATCH_SIZE] x [SEQ_LENGTH] x [ITEM_SIZE] (agnostic to item size)")
}
def output_data_definitions(self):
"""
Function returns a dictionary with definitions of output data produced the component.
:return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).
"""
return {
self.key_outputs: DataDefinition([-1, self.bow_size], [torch.Tensor], "Batch of sentences, each represented as a single vector [BATCH_SIZE x ITEM_SIZE] (agnostic to item size)")
}
