def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ input_defs = { self.key_targets: DataDefinition( [-1], [torch.Tensor], "Batch of targets, each being a single index [BATCH_SIZE]") } if self.use_prediction_distributions: input_defs[self.key_predictions] = DataDefinition([-1, -1], [ torch.Tensor ], "Batch of predictions, represented as tensor with probability distribution over classes [BATCH_SIZE x NUM_CLASSES]" ) else: input_defs[self.key_predictions] = DataDefinition([-1], [ torch.Tensor ], "Batch of predictions, represented as tensor with indices of predicted answers [BATCH_SIZE]" ) if self.use_masking: input_defs[self.key_masks] = DataDefinition( [-1], [torch.Tensor], "Batch of masks [BATCH_SIZE]") return input_defs
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ if self.mode_reverse: if self.use_input_distributions: return { self.key_inputs: DataDefinition([-1, -1, -1], [ torch.Tensor ], "Batch of sentences represented as a single tensor with batch of probability distributions [BATCH_SIZE x SEQ_LENGTH x ITEM_SIZE]" ), } else: return { self.key_inputs: DataDefinition([-1, -1], [ torch.Tensor ], "Batch of sentences represented as a single tensor of indices of particular words [BATCH_SIZE x SEQ_LENGTH]" ), } else: return { self.key_inputs: DataDefinition([-1, -1, 1], [ list, list, str ], "Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]" ), }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ if self.return_feature_maps: return { self.key_outputs: DataDefinition([ -1, self.feature_maps_depth, self.feature_maps_height, self.feature_maps_width ], [ torch.Tensor ], "Batch of feature maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]" ) } else: return { self.key_outputs: DataDefinition([-1, self.output_size], [ torch.Tensor ], "Batch of outputs, each represented as probability distribution over classes [BATCH_SIZE x PREDICTION_SIZE]" ) }
def input_data_definitions(self): return { "input1": DataDefinition([-1, 1], [list, int], "comment1"), "input2": DataDefinition([-1, -1, -1], [list, list, str], "comment2"), "input3": DataDefinition([-1, -1], [float], "comment3") }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ # Add targets. input_defs = { self.key_targets: DataDefinition([-1, -1], [ torch.Tensor ], "Batch of sentences represented as a single tensor of indices of particular words [BATCH_SIZE x SEQ_LENGTH]" ), } # Add predictions. if self.use_prediction_distributions: input_defs[self.key_predictions] = DataDefinition([-1, -1, -1], [ torch.Tensor ], "Batch of predictions, represented as tensor with sequences of probability distributions over classes [BATCH_SIZE x SEQ_LENGTH x NUM_CLASSES]" ) else: input_defs[self.key_predictions] = DataDefinition([-1, -1], [ torch.Tensor ], "Batch of predictions, represented as tensor with sequences of indices of predicted answers [BATCH_SIZE x SEQ_LENGTH]" ) # Add masks. #if self.use_masking: # input_defs[self.key_masks] = DataDefinition([-1, -1], [torch.Tensor], "Batch of masks (separate mask for each sequence in the batch) [BATCH_SIZE x SEQ_LENGTH]") return input_defs
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"), self.key_inputs: DataDefinition([-1, 1, self.height, self.width], [ torch.Tensor ], "Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]" ), self.key_targets: DataDefinition( [-1], [torch.Tensor], "Batch of targets, each being a single index [BATCH_SIZE]"), self.key_labels: DataDefinition([-1, 1], [ list, str ], "Batch of targets, each being a single word [BATCH_SIZE] x [STRING]" ) }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`). """ return { self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"), self.key_images: DataDefinition([-1, -1, -1, -1], [ torch.Tensor ], "Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]" ), self.key_labels: DataDefinition([-1, 1], [ list, str ], "Batch of target labels, each being a single word [BATCH_SIZE] x [STRING]" ), self.key_answers: DataDefinition([-1, 1], [ list, str ], "Batch of predicted labels, each being a single word [BATCH_SIZE] x [STRING]" ) }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ d = {} if self.prediction_mode == "Dense": d[self.key_predictions] = DataDefinition([ -1, -1, self.prediction_size ], [ torch.Tensor ], "Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]" ) elif self.prediction_mode == "Last": # "Last" # Only last prediction. d[self.key_predictions] = DataDefinition([ -1, self.prediction_size ], [ torch.Tensor ], "Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]" ) # Output hidden state stream TODO: why do we need that? if self.output_last_state: d[self.key_output_state] = DataDefinition([ -1, 1, self.hidden_size ], [ torch.Tensor ], "Batch of RNN final hidden states [BATCH_SIZE x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]" ) return d
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_feature_maps: DataDefinition([-1, self.feature_maps_depth, self.feature_maps_height, self.feature_maps_width], [torch.Tensor], "Batch of feature maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]"), self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], "Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]"), }
def output_data_definitions(self): """ Function returns a empty dictionary with definitions of output data produced the component. :return: Empty dictionary. """ return { self.key_masks: DataDefinition([-1], [torch.Tensor], "Batch of masks [BATCH_SIZE]"), self.key_string_indices: DataDefinition([-1], [torch.Tensor], "Batch of indices corresponging to inputs strings when using provided word mappings [BATCH_SIZE]") }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_image_encodings: DataDefinition([-1, self.image_encoding_size], [torch.Tensor], "Batch of encoded images [BATCH_SIZE x IMAGE_ENCODING_SIZE]"), self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], "Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]"), }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_targets: DataDefinition([-1]*self.num_targets_dims, [torch.Tensor], "Batch of targets (indices) [DIM 1 x DIM 2 x ... ]"), self.key_predictions: DataDefinition([-1]*(self.num_targets_dims+1), [torch.Tensor], "Batch of predictions, represented as tensor with probability distribution over classes [DIM 1 x DIM x ... x NUM_CLASSES]") }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ if self.mode_detokenize == False: return { self.key_outputs: DataDefinition([-1, -1, 1], [list, list, str], "Batch of tokenized sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]") } else: return { self.key_outputs: DataDefinition([-1, 1], [list, str], "Batch of sentences, each represented as a single string [BATCH_SIZE] x [string]") }
def test_override(self): """ Tests whether values cannot be overriden. """ dd = DataDefinition([1], [int], "Value") with self.assertRaises(AttributeError): dd.dimensions = [1, 2] with self.assertRaises(AttributeError): dd.types = [str, list] with self.assertRaises(AttributeError): dd.description = "New Description"
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"), self.key_sources: DataDefinition([-1, self.sentence_length, 1], [list, list, str], "Batch of input sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]"), self.key_targets: DataDefinition([-1, self.sentence_length, 1], [list, list, str], "Batch of target sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]") }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"), #self.key_inputs: DataDefinition([-1, self.context, 1], [list, list, str], "Batch of sentences, each being a context consisint of several words [BATCH_SIZE] x [CONTEXT_SIZE] x [WORD]"), self.key_inputs: DataDefinition([-1, 1], [list, str], "Batch of sentences, each being a context consisint of several words [BATCH_SIZE] x [string CONTEXT_SIZE * WORD]"), self.key_targets: DataDefinition([-1, 1], [list, str], "Batch of targets, each being a single word [BATCH_SIZE] x [WORD]") }
def test_values(self): """ Tests whether the values are set. """ dd = DataDefinition([1], [int], "Value") self.assertEqual(dd.dimensions, [1]) self.assertEqual(dd.types, [int]) self.assertEqual(dd.description, "Value")
def output_data_definitions(self): """ Function returns a empty dictionary with definitions of output data produced the component. :return: Empty dictionary. """ return { self.key_output_indices: DataDefinition([-1], [torch.Tensor], "Batch of merged (output) indices [BATCH_SIZE]"), self.key_output_strings: DataDefinition([-1, 1], [ list, str ], "Batch of merged strings, corresponging to indices when using the provided word mappings [BATCH_SIZE] x [STRING]" ) }
def test_extension_definitions(self): """ Tests extension of output definition keys. """ all_defs = {} # Key not existing in output definitions - ADD. all_defs["output2"] = DataDefinition([-1, -1, -1], [list, list, str], "comment") self.assertEqual( self.component.export_output_definitions(all_defs, log_errors=False), 0) # Key already existing in output definitions. all_defs["output"] = DataDefinition([-1, -1, -1], [list, list, str], "comment") self.assertEqual( self.component.export_output_definitions(all_defs, log_errors=False), 1)
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_inputs: DataDefinition([-1, -1, 1], [list, list, str], "Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]") }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_outputs: DataDefinition([-1, -1, self.embeddings_size], [torch.Tensor], "Batch of embedded sentences [BATCH_SIZE x SENTENCE_LENGTH x EMBEDDING_SIZE]") }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_inputs: DataDefinition([-1, self.input_size], [torch.Tensor], "Batch of inputs, each represented as index [BATCH_SIZE x INPUT_SIZE]"), }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of indices [BATCH_SIZE] x [1]") }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_predictions: DataDefinition([-1, self.prediction_size], [torch.Tensor], "Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x PREDICTION_SIZE]") }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], "Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]") }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`). """ return { self.key_strings: DataDefinition([-1, 1], [list, str], "Batch of strings, each being treated as a single 'vocabulary entry' (word) [BATCH_SIZE] x [STRING]") }
def test_handshake_input_definitions_types(self): """ Tests handskaking of input definition keys. """ all_defs = {} all_defs["input2"] = DataDefinition([-1, -1, -1], [list, list, str], "comment2") all_defs["input3"] = DataDefinition([-1, -1], [float], "comment3") # One input with wrong number of types. all_defs["input1"] = DataDefinition([-1, 1], [list], "comment") self.assertEqual( self.component.handshake_input_definitions(all_defs, log_errors=False), 1) # One input with one wrong type. all_defs["input1"] = DataDefinition([-1, 1], [list, str], "comment") self.assertEqual( self.component.handshake_input_definitions(all_defs, log_errors=False), 1)
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_outputs: DataDefinition([-1], [torch.Tensor], "Batch of labels, each represented as a single index [BATCH_SIZE]") }
def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_inputs: DataDefinition([-1, -1, self.bow_size], [list, list, torch.Tensor], "Batch of sentences, each represented as a list of vectors [BATCH_SIZE] x [SEQ_LENGTH] x [ITEM_SIZE] (agnostic to item size)") }
def output_data_definitions(self): """ Function returns a dictionary with definitions of output data produced the component. :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). """ return { self.key_outputs: DataDefinition([-1, self.bow_size], [torch.Tensor], "Batch of sentences, each represented as a single vector [BATCH_SIZE x ITEM_SIZE] (agnostic to item size)") }