def test_should_not_fail_on_shuffle_dataset(self, preprocessor, embeddings): data_generator = DataGenerator(np.asarray([SENTENCE_TOKENS_1]), np.asarray([[LABEL_1]]), features=np.asarray( [SENTENCE_FEATURES_1]), preprocessor=preprocessor, embeddings=embeddings, **DEFAULT_ARGS) data_generator._shuffle_dataset() # pylint: disable=protected-access
def test_should_return_left_pad_batch_values(self, preprocessor, embeddings): preprocessor.return_features = True batch_size = 2 batch = DataGenerator( np.asarray([LONG_SENTENCE_TOKENS, SHORT_SENTENCE_TOKENS]), np.asarray([[LABEL_1] * len(LONG_SENTENCE_TOKENS), [LABEL_2] * len(SHORT_SENTENCE_TOKENS)]), features=np.asarray([ np.asarray([WORD_FEATURES_1] * len(LONG_SENTENCE_TOKENS)), np.asarray([WORD_FEATURES_2] * len(SHORT_SENTENCE_TOKENS)) ]), preprocessor=preprocessor, embeddings=embeddings, **{ **DEFAULT_ARGS, 'batch_size': batch_size })[0] LOGGER.debug('batch: %s', batch) assert len(batch) == 2 inputs, _ = batch batch_x = inputs[0] batch_features = inputs[2] assert get_lengths(batch_x) == [len(LONG_SENTENCE_TOKENS)] * batch_size assert get_lengths(batch_features) == [len(LONG_SENTENCE_TOKENS) ] * batch_size
def test_should_truncate_using_max_sequence_length_if_already_tokenized( self, preprocessor, embeddings): preprocessor.return_features = True batch_size = 2 LOGGER.debug('SHORT_SENTENCE_TOKENS: %s', SHORT_SENTENCE_TOKENS) LOGGER.debug('LONG_SENTENCE_TOKENS: %s', LONG_SENTENCE_TOKENS) batch = DataGenerator( np.asarray([LONG_SENTENCE_TOKENS, SHORT_SENTENCE_TOKENS]), np.asarray([[LABEL_1] * len(LONG_SENTENCE_TOKENS), [LABEL_2] * len(SHORT_SENTENCE_TOKENS)]), features=np.asarray([ np.asarray([WORD_FEATURES_1] * len(LONG_SENTENCE_TOKENS)), np.asarray([WORD_FEATURES_2] * len(SHORT_SENTENCE_TOKENS)) ]), preprocessor=preprocessor, embeddings=embeddings, max_sequence_length=len(SHORT_SENTENCE_TOKENS), **{ **DEFAULT_ARGS, 'batch_size': batch_size, 'tokenize': False })[0] LOGGER.debug('batch: %s', batch) assert len(batch) == 2 inputs, batch_y = batch batch_x = inputs[0] batch_features = inputs[2] assert get_lengths(batch_x) == [len(SHORT_SENTENCE_TOKENS) ] * batch_size assert get_lengths(batch_features) == [len(SHORT_SENTENCE_TOKENS) ] * batch_size assert get_lengths(batch_y) == [len(SHORT_SENTENCE_TOKENS) ] * batch_size
def test_should_concatenate_word_embeddings_when_using_text_feature( self, preprocessor, embeddings): preprocessor.return_casing = False sentence_tokens_1 = [WORD_1, WORD_2] text_1 = ' '.join([WORD_3, WORD_4]) text_2 = ' '.join([WORD_4, PAD]) features_1 = [[[text_1], [text_2]]] expected_char_indices_1 = get_words_char_indices([text_1, text_2]) # by default only using the first word token for embeddings expected_word_vectors_1 = np.concatenate([ get_word_vectors([WORD_3, WORD_4]), get_word_vectors([WORD_4, PAD]) ], axis=-1) item = DataGenerator(np.asarray([sentence_tokens_1]), np.asarray([[LABEL_1]]), features=np.asarray(features_1), preprocessor=preprocessor, embeddings=embeddings, text_feature_indices=[0], concatenated_embeddings_token_count=2, **DEFAULT_ARGS)[0] LOGGER.debug('item: %s', item) assert len(item) == 2 x, labels = item assert all_close(labels, get_label_indices([LABEL_1])) assert all_close(x[0], expected_word_vectors_1) LOGGER.debug('expected char_indices_1: %s', expected_char_indices_1) LOGGER.debug('x[1]: %s', x[1]) assert all_close(x[1], expected_char_indices_1) assert all_close(x[-1], [len(sentence_tokens_1)])
def test_should_concatenate_word_embeddings_if_using_multiple_tokens( self, preprocessor, embeddings): preprocessor.return_casing = False sentence_tokens_1 = [WORD_1, WORD_2] feature_tokens_1 = [WORD_3, WORD_4] item = DataGenerator(np.asarray([sentence_tokens_1]), np.asarray([[LABEL_1]]), features=np.asarray([[[WORD_3], [WORD_4]]]), additional_token_feature_indices=[0], preprocessor=preprocessor, embeddings=embeddings, **DEFAULT_ARGS)[0] LOGGER.debug('item: %s', item) assert len(item) == 2 x, labels = item assert all_close(labels, get_label_indices([LABEL_1])) assert all_close( x[0], np.concatenate((get_word_vectors(sentence_tokens_1), get_word_vectors(feature_tokens_1)), axis=-1)) assert all_close( x[1], np.concatenate(([get_words_char_indices(sentence_tokens_1) ], [get_words_char_indices(feature_tokens_1)]), axis=-1)) assert all_close(x[-1], [len(sentence_tokens_1)])
def test_should_be_able_to_get_item(self, preprocessor, embeddings): item = DataGenerator(np.asarray([SENTENCE_TOKENS_1]), np.asarray([[LABEL_1]]), preprocessor=preprocessor, embeddings=embeddings, **DEFAULT_ARGS)[0] LOGGER.debug('item: %s', item) assert len(item) == 2 x, labels = item assert all_close(labels, get_label_indices([LABEL_1])) assert all_close(x[0], get_word_vectors(SENTENCE_TOKENS_1)) assert all_close(x[1], [get_words_char_indices(SENTENCE_TOKENS_1)]) assert all_close(x[-1], [len(SENTENCE_TOKENS_1)])
def test_should_return_features(self, preprocessor, embeddings): preprocessor.return_features = True item = DataGenerator(np.asarray([SENTENCE_TOKENS_1]), np.asarray([[LABEL_1]]), features=np.asarray([SENTENCE_FEATURES_1]), preprocessor=preprocessor, embeddings=embeddings, **DEFAULT_ARGS)[0] LOGGER.debug('item: %s', item) assert len(item) == 2 x, labels = item assert all_close(labels, get_label_indices([LABEL_1])) assert all_close(x[0], get_word_vectors(SENTENCE_TOKENS_1)) assert all_close(x[1], [get_words_char_indices(SENTENCE_TOKENS_1)]) assert all_close(x[2], [get_transformed_features(SENTENCE_FEATURES_1)]) assert all_close(x[-1], [len(SENTENCE_TOKENS_1)])
def test_should_use_dummy_word_embeddings_if_disabled(self, preprocessor): preprocessor.return_casing = False item = DataGenerator(np.asarray([SENTENCE_TOKENS_1]), np.asarray([[LABEL_1]]), preprocessor=preprocessor, use_word_embeddings=False, embeddings=None, **DEFAULT_ARGS)[0] LOGGER.debug('item: %s', item) assert len(item) == 2 x, labels = item assert all_close(labels, get_label_indices([LABEL_1])) assert all_close( x[0], [np.zeros((len(SENTENCE_TOKENS_1), 0), dtype='float32')]) assert all_close(x[1], [get_words_char_indices(SENTENCE_TOKENS_1)]) assert all_close(x[-1], [len(SENTENCE_TOKENS_1)])
def create_data_generator(self, *args, name_suffix: str, **kwargs) -> DataGenerator: return DataGenerator( # type: ignore *args, batch_size=self.training_config.batch_size, input_window_stride=self.training_config.input_window_stride, stateful=self.model_config.stateful, preprocessor=self.preprocessor, additional_token_feature_indices=self.model_config. additional_token_feature_indices, text_feature_indices=self.model_config.text_feature_indices, concatenated_embeddings_token_count=( self.model_config.concatenated_embeddings_token_count), char_embed_size=self.model_config.char_embedding_size, is_deprecated_padded_batch_text_list_enabled=( self.model_config.is_deprecated_padded_batch_text_list_enabled ), max_sequence_length=self.model_config.max_sequence_length, embeddings=self.embeddings, name='%s.%s' % (self.model_config.model_name, name_suffix), **kwargs)
def create_eval_data_generator(self, *args, **kwargs) -> DataGenerator: return DataGenerator( # type: ignore *args, batch_size=( self.eval_batch_size or self.training_config.batch_size ), preprocessor=self.p, additional_token_feature_indices=self.model_config.additional_token_feature_indices, text_feature_indices=self.model_config.text_feature_indices, concatenated_embeddings_token_count=( self.model_config.concatenated_embeddings_token_count ), char_embed_size=self.model_config.char_embedding_size, is_deprecated_padded_batch_text_list_enabled=( self.model_config.is_deprecated_padded_batch_text_list_enabled ), max_sequence_length=self.eval_max_sequence_length, embeddings=self.embeddings, **kwargs )
def test_should_generate_windows_and_disable_shuffle( self, preprocessor, embeddings): batches = DataGenerator(np.asarray([[WORD_1, WORD_2, WORD_3]]), np.asarray([[LABEL_1, LABEL_2, LABEL_3]]), preprocessor=preprocessor, embeddings=embeddings, input_window_stride=2, max_sequence_length=2, **{ **DEFAULT_ARGS, 'shuffle': True }) assert not batches.shuffle LOGGER.debug('batches: %s', batches) assert len(batches) == 2 batch_0 = batches[0] LOGGER.debug('batch_0: %s', batch_0) assert len(batch_0) == 2 x, labels = batch_0 LOGGER.debug('labels: %s', labels) assert all_close(labels, get_label_indices([LABEL_1, LABEL_2])) assert all_close(x[0], get_word_vectors([WORD_1, WORD_2])) assert all_close(x[1], [get_words_char_indices([WORD_1, WORD_2])]) assert all_close(x[-1], [2]) batch_1 = batches[1] LOGGER.debug('batch_1: %s', batch_1) # due to extend, the minimum length is 2 assert len(batch_1) == 2 x, labels = batch_1 LOGGER.debug('labels: %s', labels) assert all_close(labels, get_label_indices([LABEL_3])) assert all_close(x[0], get_word_vectors([WORD_3, None])) assert all_close(x[1], [get_words_char_indices([WORD_3, None])]) assert all_close(x[-1], [1])
def iter_predict_texts_with_sliding_window_if_enabled( texts: List[Union[str, List[str]]], model_config: ModelConfig, preprocessor: WordPreprocessor, max_sequence_length: Optional[int], model, input_window_stride: int = None, embeddings: Embeddings = None, features: List[List[List[str]]] = None): if not texts: LOGGER.info('passed in empty texts, model: %s', model_config.model_name) return should_tokenize = ( len(texts) > 0 # pylint: disable=len-as-condition and isinstance(texts[0], str)) if not should_tokenize and max_sequence_length: max_actual_sequence_length = max(len(text) for text in texts) if max_actual_sequence_length <= max_sequence_length: LOGGER.info( 'all text sequences below max sequence length: %d <= %d (model: %s)', max_actual_sequence_length, max_sequence_length, model_config.model_name) elif model_config.stateful: LOGGER.info(( 'some text sequences exceed max sequence length (using sliding windows):' ' %d > %d (model: %s)'), max_actual_sequence_length, max_sequence_length, model_config.model_name) else: LOGGER.info( ('some text sequences exceed max sequence length' ' (truncate, model is not stateful): %d > %d (model: %s)'), max_actual_sequence_length, max_sequence_length, model_config.model_name) predict_generator = DataGenerator( x=texts, y=None, batch_size=model_config.batch_size, preprocessor=preprocessor, additional_token_feature_indices=model_config. additional_token_feature_indices, text_feature_indices=model_config.text_feature_indices, concatenated_embeddings_token_count=( model_config.concatenated_embeddings_token_count), char_embed_size=model_config.char_embedding_size, is_deprecated_padded_batch_text_list_enabled=( model_config.is_deprecated_padded_batch_text_list_enabled), max_sequence_length=max_sequence_length, input_window_stride=input_window_stride, stateful=model_config.stateful, embeddings=embeddings, tokenize=should_tokenize, shuffle=False, features=features, name='%s.predict_generator' % model_config.model_name) prediction_list_list: List[List[np.ndarray]] = [[] for _ in texts] batch_window_indices_and_offsets_iterable = logging_tqdm( iter_batch_window_indices_and_offsets(predict_generator), logger=LOGGER, total=len(predict_generator), desc='%s: ' % predict_generator.name, unit='batch') completed_curser = 0 for batch_window_indices_and_offsets in batch_window_indices_and_offsets_iterable: LOGGER.debug('predict batch_window_indices_and_offsets: %s', batch_window_indices_and_offsets) generator_output = predict_generator.get_window_batch_data( batch_window_indices_and_offsets) LOGGER.debug('predict on batch: %d', len(batch_window_indices_and_offsets)) batch_predictions = model.predict_on_batch(generator_output[0]) LOGGER.debug('preds.shape: %s', batch_predictions.shape) for window_indices_and_offsets, seq_predictions in zip( batch_window_indices_and_offsets, batch_predictions): text_index, text_offset = window_indices_and_offsets current_prediction_list = prediction_list_list[text_index] LOGGER.debug('prediction_list_list[%d]: %s', text_index, current_prediction_list) current_offset = sum((len(a) for a in current_prediction_list)) if current_offset > text_offset: # skip over the overlapping window seq_predictions = seq_predictions[(current_offset - text_offset):, :] text_offset = current_offset assert (current_offset == text_offset ), "expected %d to be %d" % (current_offset, text_offset) current_prediction_list.append(seq_predictions) next_offset = sum((len(a) for a in current_prediction_list)) is_complete = (next_offset >= len(texts[text_index])) LOGGER.debug( 'is_complete: %s, text_index=%d, completed_curser=%d, next_offset=%d, textlen=%d', is_complete, text_index, completed_curser, next_offset, len(texts[text_index])) if (is_complete and text_index == completed_curser): yield np.concatenate(current_prediction_list, axis=0) completed_curser += 1 for prediction_list in prediction_list_list[completed_curser:]: yield np.concatenate(prediction_list, axis=0)
def iter_batch_window_indices_and_offsets( data_generator: DataGenerator) -> Iterable[List[Tuple[int, int]]]: return (data_generator.get_batch_window_indices_and_offsets(batch_index) for batch_index in range(len(data_generator)))
def test_should_be_able_to_instantiate(self, preprocessor, embeddings): DataGenerator(np.asarray([[WORD_1, WORD_2]]), np.asarray([[LABEL_1]]), preprocessor=preprocessor, embeddings=embeddings, **DEFAULT_ARGS)