def loadData(): raw_train_ds = text_dataset_from_directory(path_to_train_file, batch_size=batch_size) raw_test_ds = text_dataset_from_directory(path_to_test_file, batch_size=batch_size) print("train_size = {0} test_size = {1}".format(len(list(raw_train_ds)), len(list(raw_test_ds)))) length_min = 1000 length_max = 0 all_length = [] for element in raw_train_ds.as_numpy_iterator(): sentences = element[0] for s in sentences: l = len(s.decode("utf-8").split(" ")) if l < length_min: length_min = l if l > length_max: length_max = l all_length.append(l) print("length_min = {0} length_max = {1}".format(length_min, length_max)) plt.figure() plt.hist(all_length, bins=500) plt.xlim(left=0) plt.title("Histogram of sequance length") plt.xlabel("length bins") plt.ylabel("number of words") plt.figure() plt.title("Boxplot of sequance length") plt.boxplot(all_length, labels=['number of words']) return raw_train_ds, raw_test_ds
def make_csv(self, data_input_path, data_output_path, use_saved_tokenizer=False): dataset = text_dataset_from_directory( data_input_path, label_mode="categorical", batch_size=self.batch_size, max_length=self.max_char_length, shuffle=True, ) with open(data_output_path, "w") as data_file: writer = csv.writer(data_file) if self.tokenizer_type == BERT: sequencer = BertSequencer() self._prepare_and_write(dataset, sequencer, writer) elif self.tokenizer_type == CUSTOM: sequencer = CustomSequencer() if use_saved_tokenizer: sequencer.tokenizer = CustomSequencer.load_tokenizer( self.tokenizer_path) else: for batch in dataset: sentences, labels = batch sentences = [ sentence.decode("utf-8") for sentence in sentences.numpy() ] sequencer.feed_tokenizer(sentences) sequencer.save_tokenizer(self.tokenizer_path) self._prepare_and_write(dataset, sequencer, writer)
def datset_objbuilder(main_dir_path, debuggining_info=True): """ :param main_dir_path: Path to main directory :return: """ complete_dataset = text_dataset_from_directory( main_dir_path, class_names=['neg', 'pos'], ) if debuggining_info: print("Accessing and Building Dataset....") print("First_batch_of_dataset..") print(list(complete_dataset.take( 1).as_numpy_iterator())) # 32 files (batch_size) print(complete_dataset.element_spec) print("Dataset Building Successful") for files, targets in complete_dataset.take(1).as_numpy_iterator(): print("First File content: ") print(files[0]) print("Target: ") print(targets[0]) break return complete_dataset
list(dataset_dir.iterdir()) train_dir = dataset_dir / 'train' print(list(train_dir.iterdir())) sample_file = train_dir / 'python/1755.txt' with open(sample_file) as f: print(f.read()) batch_size = 32 seed = 42 raw_train_ds = preprocessing.text_dataset_from_directory(train_dir, batch_size=batch_size, validation_split=0.2, subset='training', seed=seed) for text_batch, label_batch in raw_train_ds.take(1): for i in range(10): print("Question: ", text_batch.numpy()[i][:100], '...') print("Label:", label_batch.numpy()[i]) for i, label in enumerate(raw_train_ds.class_names): print("Label", i, "corresponds to", label) raw_val_ds = preprocessing.text_dataset_from_directory(train_dir, batch_size=batch_size, validation_split=0.2, subset='validation',
import tensorflow as tf import re import string from tensorflow.keras import layers, losses from tensorflow.keras.layers.experimental.preprocessing import TextVectorization from tensorflow.keras.preprocessing import text_dataset_from_directory if not tf.config.list_physical_devices('GPU'): print('No GPU is available') else: tf.config.experimental.set_memory_growth( tf.config.list_physical_devices('GPU')[0], True) train_ds = text_dataset_from_directory( './stack_overflow_16k/train', batch_size=32, validation_split=0.2, subset='training', seed=42).cache().prefetch(buffer_size=tf.data.AUTOTUNE) val_ds = text_dataset_from_directory( './stack_overflow_16k/train', batch_size=32, validation_split=0.2, subset='validation', seed=42).cache().prefetch(buffer_size=tf.data.AUTOTUNE) test_ds = text_dataset_from_directory( './stack_overflow_16k/train', batch_size=32).cache().prefetch(buffer_size=tf.data.AUTOTUNE) def custom_standardization(input_data): lowercase = tf.strings.lower(input_data)
def prepareData(dir): data = pp.text_dataset_from_directory(dir) return data.map( lambda text, label: (strings.regex_replace(text, '<br />', ' '), label), )
train_dir = 'testingoutput' val_dir = 'expected_text_result' test_dir = 'test_data' batch_size = 32 seed = 42 labels =['home depot', 'walmart', 'Tea', 'safeway', 'walmart', 'walmart', 'walmart'] train_ds = preprocessing.text_dataset_from_directory(train_dir, batch_size=batch_size, seed=seed) val_ds = preprocessing.text_dataset_from_directory(val_dir, batch_size=batch_size, seed=seed) test_ds = preprocessing.text_dataset_from_directory(test_dir, batch_size=batch_size, seed=seed) VOCAB_SIZE = 10000 binary_vectorize_layer = TextVectorization( max_tokens=VOCAB_SIZE, output_mode='binary')
# In[3]: from tensorflow.keras.preprocessing import text_dataset_from_directory import pandas as pd from sklearn.model_selection import train_test_split # In[4]: # Importing & preprocessing the dataset train_ds = text_dataset_from_directory('../neuralnets/aclImdb/train') test_ds = text_dataset_from_directory('../neuralnets/aclImdb/test') dfTrain = pd.DataFrame(train_ds.unbatch().as_numpy_iterator(), columns=['text', 'label']) dfTest = pd.DataFrame(test_ds.unbatch().as_numpy_iterator(), columns=['text', 'label']) _, xts = train_test_split(dfTest, stratify=dfTest['label'], test_size=0.25) # In[5]: len(model.wv) # In[6]:
# remove unecessary directories if os.path.exists(os.path.join(train_dir, "unsup")): remove_dir = os.path.join(train_dir, "unsup") shutil.rmtree(remove_dir) # c'est recommandé de diviser l'ensemble de données en 3 # divisons (entrainement, validation et test) # création de l'ensemble de validation batch_size = 32 seed = 42 raw_train_ds = preprocessing.text_dataset_from_directory( "aclImdb/train", batch_size=batch_size, validation_split=0.2, subset="training", seed=seed, ) # for text_batch, label_batch in raw_train_ds.take(1): # for i in range(3): # print('Review', text_batch.numpy()[i]) # print('Label', raw_train_ds.class_names[label_batch.numpy()[i]]) raw_val_ds = preprocessing.text_dataset_from_directory( "aclImdb/train", batch_size=batch_size, validation_split=0.2, subset="validation", seed=seed,
import tensorflow as tf import numpy as np import sys import os from tensorflow.keras.layers.experimental.preprocessing import TextVectorization from tensorflow.keras import layers from tensorflow.keras.preprocessing import text_dataset_from_directory training_dir = sys.argv[1] batch_size = 32 raw_train_ds = text_dataset_from_directory( training_dir, batch_size=batch_size, validation_split=0.2, subset="training", seed=1337, ) raw_val_ds = text_dataset_from_directory( training_dir, batch_size=batch_size, validation_split=0.2, subset="validation", seed=1337, ) raw_test_ds = text_dataset_from_directory( training_dir, batch_size=batch_size*2 ) max_features = 20000 embedding_dim = 128
def main(): # Example 1: Predict the tag for a Stack Overflow question # Download and explore the dataset url = 'https://storage.googleapis.com/download.tensorflow.org/data/stack_overflow_16k.tar.gz' dataset = utils.get_file('stack_overflow_16k.tar.gz', url, untar=True, cache_dir='stack_overflow', cache_subdir='') dataset_dir = pathlib.Path(dataset).parent print(f'Paths: {list(dataset_dir.iterdir())}') train_dir = dataset_dir / 'train' print(f'Train paths: {train_dir.iterdir}') sample_file = train_dir / 'python/1755.txt' with open(sample_file) as f: print(f.read()) # Load the dataset batch_size = 32 seed = 42 raw_train_ds = preprocessing.text_dataset_from_directory( train_dir, batch_size=batch_size, validation_split=0.2, subset='training', seed=seed) for text_batch, label_batch in raw_train_ds.take(1): for i in range(10): print(f'Question: {text_batch.numpy()[i]}') print(f'Label: {label_batch.numpy()[i]}') for i, label in enumerate(raw_train_ds.class_names): print(f'Label {i} corresponds to {label}') raw_val_ds = preprocessing.text_dataset_from_directory( train_dir, batch_size=batch_size, validation_split=0.2, subset='validation', seed=seed) test_dir = dataset_dir / 'test' raw_test_ds = preprocessing.text_dataset_from_directory( test_dir, batch_size=batch_size) # Prepare the dataset for training vocab_size = 10000 binary_vectorize_layer = TextVectorization(max_tokens=vocab_size, output_mode='binary') max_sequence_length = 250 int_vectorize_layer = TextVectorization( max_tokens=vocab_size, output_mode='int', output_sequence_length=max_sequence_length) # Make a text-only dataset (without labels), then call adapt train_text = raw_train_ds.map(lambda text, labels: text) binary_vectorize_layer.adapt(train_text) int_vectorize_layer.adapt(train_text) def binary_vectorize_text(text, label): text = tf.expand_dims(text, -1) return binary_vectorize_layer(text), label def int_vectorize_text(text, label): text = tf.expand_dims(text, -1) return int_vectorize_layer(text), label # Retrieve a batch (of 32 reviews and labels) from the dataset text_batch, label_batch = next(iter(raw_train_ds)) first_question, first_label = text_batch[0], label_batch[0] print(f'Question: {first_question}') print(f'Label: {first_label}') print( f'"binary" vectorized question: {binary_vectorize_text(first_question, first_label)[0]}' ) print( f'"int" vectorized question: {int_vectorize_text(first_question, first_label)[0]}' ) print(f'1298 ---> {int_vectorize_layer.get_vocabulary()[1289]}') print(f' 313 ---> {int_vectorize_layer.get_vocabulary()[313]}') print(f'Vocabulary size: {len(int_vectorize_layer.get_vocabulary())}') binary_train_ds = raw_train_ds.map(binary_vectorize_text) binary_val_ds = raw_val_ds.map(binary_vectorize_text) binary_test_ds = raw_test_ds.map(binary_vectorize_text) int_train_ds = raw_train_ds.map(int_vectorize_text) int_val_ds = raw_val_ds.map(int_vectorize_text) int_test_ds = raw_test_ds.map(int_vectorize_text) # Configure the dataset for performance AUTOTUNE = tf.data.AUTOTUNE def configure_dataset(dataset): return dataset.cache().prefetch(buffer_size=AUTOTUNE) binary_train_ds = configure_dataset(binary_train_ds) binary_val_ds = configure_dataset(binary_val_ds) binary_test_ds = configure_dataset(binary_test_ds) int_train_ds = configure_dataset(int_train_ds) int_val_ds = configure_dataset(int_val_ds) int_test_ds = configure_dataset(int_test_ds) # Train the model binary_model = tf.keras.Sequential([layers.Dense(4)]) binary_model.compile( optimizer='adam', loss=losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) _ = binary_model.fit(binary_train_ds, validation_data=binary_val_ds, epochs=10) def create_model(vocab_size, num_labels): model = tf.keras.Sequential([ layers.Embedding(vocab_size, 64, mask_zero=True), layers.Conv1D(64, 5, padding='valid', activation='relu', strides=2), layers.GlobalMaxPool1D(), layers.Dense(num_labels) ]) return model # vocab_size is vocab_size + 1 since 0 is used additionally for padding. int_model = create_model(vocab_size=vocab_size + 1, num_labels=4) int_model.compile( optimizer='adam', loss=losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) _ = int_model.fit(int_train_ds, validation_data=int_val_ds, epochs=5) print(f'Linear model on binary vectorized data: {binary_model.summary()}') print(f'ConvNet model on int vectorized data: {int_model.summary()}') _, binary_accuracy = binary_model.evaluate(binary_test_ds) _, int_accuracy = int_model.evaluate(int_test_ds) print(f'Binary model accuracy {binary_accuracy:2.2%}') print(f'Int model accuracy: {int_accuracy:2.2%}') # Export the model export_model = tf.keras.Sequential( [binary_vectorize_layer, binary_model, layers.Activation('sigmoid')]) export_model.compile( optimizer='adam', loss=losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) # Test it with `raw_test_ds`, which yields raw strings loss, accuracy = export_model.evaluate(raw_test_ds) print(f'Accuracy: {accuracy:2.2%}') def get_string_labels(predicted_scores_batch): predicted_int_labels = tf.argmax(predicted_scores_batch, axis=1) predicted_labels = tf.gather(raw_train_ds.class_names, predicted_int_labels) return predicted_labels # Run inference on new data inputs = [ 'how do I extract keys from a dict into a list?', # python 'debug public static void main(string[] args) {...}', # java ] predicted_scores = export_model.predict(inputs) predicted_labels = get_string_labels(predicted_scores) for input, label in zip(inputs, predicted_labels): print(f'Question: {input}') print(f'Predicted label: {label.numpy()}') # Example 2: Predict the author of Illiad translations url = 'https://storage.googleapis.com/download.tensorflow.org/data/illiad/' file_names = ['cowper.txt', 'derby.txt', 'butler.txt'] for name in file_names: text_dir = utils.get_file(name, origin=url + name) parent_dir = pathlib.Path(text_dir).parent print(f'Paths: {list(parent_dir.iterdir())}') # Load the dataset def labeler(example, index): return example, tf.cast(index, tf.int64) labeled_data_sets = [] for i, file_name in enumerate(file_names): lines_dataset = tf.data.TextLineDataset(str(parent_dir / file_name)) labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i)) labeled_data_sets.append(labeled_dataset) buffer_size = 50000 batch_size = 64 validation_size = 5000 all_labeled_data = labeled_data_sets[0] for labeled_dataset in labeled_data_sets[1:]: all_labeled_data = all_labeled_data.concatenate(labeled_dataset) all_labeled_data = all_labeled_data.shuffle(buffer_size, reshuffle_each_iteration=False) for text, label in all_labeled_data.take(10): print(f'Sentence: {text.numpy()}') print(f'Label: {label.numpy()}') # Prepare the dataset for training tokenizer = tf_text.UnicodeScriptTokenizer() def tokenize(text, unused_label): lower_case = tf_text.case_fold_utf8(text) return tokenizer.tokenize(lower_case) tokenized_ds = all_labeled_data.map(tokenize) for text_batch in tokenized_ds.take(5): print(f'Tokens: {text_batch.numpy()}') tokenized_ds = configure_dataset(tokenized_ds) vocab_dict = collections.defaultdict(lambda: 0) for toks in tokenized_ds.as_numpy_iterator(): for tok in toks: vocab_dict[tok] += 1 vocab = sorted(vocab_dict.items(), key=lambda x: x[1], reverse=True) vocab = [token for token, count in vocab] vocab = vocab[:vocab_size] vocab_size = len(vocab) print(f'Vocab size: {vocab_size}') print(f'First five vocab entries: {vocab[:5]}') keys = vocab values = range(2, len(vocab) + 2) # reserve 0 for padding, 1 for OOV init = tf.lookup.KeyValueTensorInitializer(keys, values, key_dtype=tf.string, value_dtype=tf.int64) num_oov_buckets = 1 vocab_table = tf.lookup.StaticVocabularyTable(init, num_oov_buckets) def preprocess_text(text, label): standardized = tf_text.case_fold_utf8(text) tokenized = tokenizer.tokenize(standardized) vectorized = vocab_table.lookup(tokenized) return vectorized, label example_text, example_label = next(iter(all_labeled_data)) print(f'Sentence: {example_text.numpy()}') vectorized_text, example_label = preprocess_text(example_text, example_label) print(f'Vectorized sentence: {vectorized_text.numpy()}') all_encoded_data = all_labeled_data.map(preprocess_text) # Split the dataset into train and test train_data = all_encoded_data.skip(validation_size).shuffle(buffer_size) validation_data = all_encoded_data.take(validation_size) train_data = train_data.padded_batch(batch_size) validation_data = validation_data.padded_batch(batch_size) sample_text, sample_labels = next(iter(validation_data)) print(f'Text batch shape: {sample_text.shape}') print(f'Label batch shape: {sample_labels.shape}') print(f'First text example: {sample_text[0]}') print(f'First label example: {sample_labels[0]}') vocab_size += 2 # Train the model model = create_model(vocab_size=vocab_size, num_labels=3) model.compile(optimizer='adam', loss=losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) _ = model.fit(train_data, validation_data=validation_data, epochs=3) loss, accuracy = model.evaluate(validation_data) print(f'Loss: {loss}') print(f'Accuracy: {accuracy:2.2%}') # Export the model preprocess_layer = TextVectorization( max_tokens=vocab_size, standardize=tf_text.case_fold_utf8, split=tokenizer.tokenize, output_mode='int', output_sequence_length=max_sequence_length) preprocess_layer.set_vocabulary(vocab) export_model = tf.keras.Sequential( [preprocess_layer, model, layers.Activation('sigmoid')]) export_model.compile( optimizer='adam', loss=losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) # Create a test dataset of raw strings test_ds = all_labeled_data.take(validation_size).batch(batch_size) test_ds = configure_dataset(test_ds) loss, accuracy = export_model.evaluate(test_ds) print(f'Loss: {loss}') print(f'Accuracy: {accuracy:2.2%}') # Run inference on new data inputs = [ "Join'd to th' Ionians with their flowing robes,", # Label: 1 "the allies, and his armour flashed about him so that he seemed to all", # Label: 2 "And with loud clangor of his arms he fell.", # Label: 0 ] predicted_scores = export_model.predict(inputs) predicted_labels = tf.argmax(predicted_scores, axis=1) for input, label in zip(inputs, predicted_labels): print(f'Question: {input}') print(f'Predicted label: {label.numpy()}') # Download more datasets using TensorFlow Datasets (TFDS) train_ds = tfds.load('imdb_reviews', split='train', batch_size=batch_size, shuffle_files=True, as_supervised=True) val_ds = tfds.load('imdb_reviews', split='train', batch_size=batch_size, shuffle_files=True, as_supervised=True) for review_batch, label_batch in val_ds.take(1): for i in range(5): print(f'Review: {review_batch[i].numpy()}') print(f'Label: {label_batch[i].numpy()}') # Prepare thje dataset for training vectorize_layer = TextVectorization( max_tokens=vocab_size, output_mode='int', output_sequence_length=max_sequence_length) # Make a text-only dataset (without labels), then call adapt train_text = train_ds.map(lambda text, labels: text) vectorize_layer.adapt(train_text) def vectorize_text(text, label): text = tf.expand_dims(text, -1) return vectorize_layer(text), label train_ds = train_ds.map(vectorize_text) val_ds = val_ds.map(vectorize_text) train_ds = configure_dataset(train_ds) val_ds = configure_dataset(val_ds) # Train the model model = create_model(vocab_size=vocab_size + 1, num_labels=1) model.summary() model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(from_logits=True), metrics=['accuracy']) _ = model.fit(train_ds, validation_data=val_ds, epochs=3) loss, accuracy = model.evaluate(val_ds) print(f'Loss: {loss}') print(f'Accuracy: {accuracy:2.2%}') # Export the model export_model = tf.keras.Sequential( [vectorize_layer, model, layers.Activation('sigmoid')]) export_model.compile( optimizer='adam', loss=losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) # 0 ---> negative review # 1 ---> positive review inputs = [ 'This is a fantastic movie.', 'This is a bad movie.', 'This movie was bad that it was good.', 'I will never say yes to watching this movie.' ] predicted_scores = export_model.predict(inputs) predicted_labels = [int(round(x[0])) for x in predicted_scores] for input, label in zip(inputs, predicted_labels): print(f'Question: {input}') print(f'Predicted label: {label}')