def train(full_model, en_seq, fr_seq, batch_size, n_epochs=10): """ Training the model """ for ep in range(n_epochs): losses = [] for bi in range(0, en_seq.shape[0] - batch_size, batch_size): print("==================================================") en_onehot_seq = to_categorical(en_seq[bi:bi + batch_size, :], num_classes=en_vsize) fr_onehot_seq = to_categorical(fr_seq[bi:bi + batch_size, :], num_classes=fr_vsize) # decoder 输入:fr_onehot_seq[:, :-1, :] # decoder 标签:fr_onehot_seq[:, :1, :] full_model.train_on_batch( [en_onehot_seq, fr_onehot_seq[:, :-1, :]], fr_onehot_seq[:, 1:, :]) l = full_model.evaluate([en_onehot_seq, fr_onehot_seq[:, :-1, :]], fr_onehot_seq[:, 1:, :], batch_size=batch_size, verbose=0) losses.append(l) if (ep + 1) % 1 == 0: logger.info("Loss in epoch {}: {}".format(ep + 1, np.mean(losses)))
def _prepare(self, path): self.mapping = self._load_labels_mapping() self.number_of_classes = len(self.mapping) data = loadmat(path / "emnist-byclass.mat") # load training dataset x_train = data["dataset"][0][0][0][0][0][0].astype(np.float32) x_train = x_train.reshape(x_train.shape[0], 28, 28, 1, order="A") y_train = data["dataset"][0][0][0][0][0][1] # One hot encoding y_train = to_categorical(y_train, self.number_of_classes).astype(np.int) # Calculate mean and standard deviation for input normalization self.mean = x_train.mean() self.std = x_train.std() print("Balancing train dataset...") x_train, y_train = self._sample_to_balance(x_train, y_train) # load test dataset x_test = data["dataset"][0][0][1][0][0][0].astype(np.float32) x_test = x_test.reshape(x_test.shape[0], 28, 28, 1, order="A") y_test = data["dataset"][0][0][1][0][0][1] y_test = to_categorical(y_test, self.number_of_classes).astype(np.int) # https://www.tensorflow.org/guide/datasets self.train_dataset = tf.data.Dataset.from_tensor_slices( (x_train, y_train)) self.test_dataset = tf.data.Dataset.from_tensor_slices( (x_test, y_test)) print( f"Dataset ready, with {len(y_train)} training entries and {len(y_test)} test entries" )
def main(): #VGGもでるの読みこみ input_tensor = Input(shape=(224, 224, 3)) vgg16 = VGG16(include_top=False, weights='imagenet', input_tensor=input_tensor) model = build_VGG16_model(vgg16) model.summary() #データセットの作成 create_txt(txts[0], 'train_img\\') create_txt(txts[1], 'test_img\\') X_train, Y_train = make_dataset(txts[0]) X_test, Y_test = make_dataset(txts[1]) #前処理(正規化) X_train = X_train.astype(np.float) X_test = X_test.astype(np.float) X_train = X_train / 255 X_test = X_test / 255 Y_train = to_categorical(Y_train, 9) Y_test = to_categorical(Y_test, 9) #CNNによる学習 model, history = model_train(model, X_train, Y_train) #学習結果のグラフ描画 graph_plot(history) #テストデータの検証結果の表示 model.evaluate(X_test, Y_test) #学習モデル・重みの保存 model.save('./model/model.h5')
def train(full_model, en_seq, fr_seq, batch_size, n_epochs=10): """ Training the model """ for ep in range(n_epochs): losses = [] for bi in tqdm(range(0, en_seq.shape[0] - batch_size, batch_size)): en_onehot_seq = to_categorical(en_seq[bi:bi + batch_size, :], num_classes=en_vsize) fr_onehot_seq = to_categorical(fr_seq[bi:bi + batch_size, :], num_classes=fr_vsize) full_model.train_on_batch( [en_onehot_seq, fr_onehot_seq[:, :-1, :]], fr_onehot_seq[:, 1:, :]) l = full_model.evaluate([en_onehot_seq, fr_onehot_seq[:, :-1, :]], fr_onehot_seq[:, 1:, :], batch_size=batch_size, verbose=0) losses.append(l) if (ep + 1) % 1 == 0: logger.info("Loss in epoch {}: {}".format(ep + 1, np.mean(losses))) print('Epoch: {}, Loss: {}'.format(str(ep + 1), str(np.mean(losses)))) """ Save model """ model_save_dir = os.path.join(base_dir, 'output') if not os.path.exists(model_save_dir): os.mkdir(model_save_dir) full_model.save( os.path.join(model_save_dir, 'nmt_ep{}.h5'.format(str(ep + 1))))
def valid_generator(val_batch_size, dim='1D'): while True: ids = list(range(valid_df.shape[0])) for start in range(0, len(ids), val_batch_size): x_batch = [] x_batch_1d = [] y_batch = [] end = min(start + val_batch_size, len(ids)) i_val_batch = ids[start:end] if dim == 'combi': for i in i_val_batch: x_2d, x_1d = process_wav_file(valid_df.wav_file.values[i], phase='TRAIN', dim=dim) x_batch.append(x_2d) x_batch_1d.append(x_1d) y_batch.append(valid_df.label_id.values[i]) x_batch = np.array(x_batch) x_batch_1d = np.array(x_batch_1d) y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS)) yield [x_batch, x_batch_1d], y_batch else: for i in i_val_batch: x_batch.append( process_wav_file(valid_df.wav_file.values[i], phase='TRAIN', dim=dim)) y_batch.append(valid_df.label_id.values[i]) x_batch = np.array(x_batch) y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS)) yield x_batch, y_batch
def _ensemble_preds(image_id, save_path, binary_mask_path_list, data_key): for i, base_path in enumerate(binary_mask_path_list): # these numpy should have been saved _load_eval_w_thresh_list with save_binary=True data_path = base_path + '/' + image_id + '.npy' if i==0: if not path.exists(data_path): print('{} is not exist!!!'.format(data_path)) else: #for tta, 'pred' -> 'mean_pred'? data = np.load(data_path)[()] pred = data[data_key] pred = to_categorical(pred, num_classes=2) else: if not path.exists(data_path): print('{} is not exist!!!'.format(data_path)) else: pred_2 = np.load(data_path)[()][data_key] pred_2 = to_categorical(pred_2, num_classes=2) pred = np.add(pred, pred_2) # what if the number of models are even? pred = np.argmax(pred, axis=-1) # sve data={'pred':, 'mask': } here in order to eval with _load_eval_w_thresh_list # for test data, mask is not supplied, just save pred if 'mask' in data.keys(): new_data = {'mask':data['mask'], 'pred':pred} else: new_data = {'pred':pred} np.save(save_path + '/' + image_id, new_data)
def train_generator(train_batch_size, dim='1D'): while True: this_train = train_df.groupby('label_id').apply( lambda x: x.sample(n=2000)) shuffled_ids = random.sample(range(this_train.shape[0]), this_train.shape[0]) for start in range(0, len(shuffled_ids), train_batch_size): x_batch = [] x_batch_1d = [] y_batch = [] end = min(start + train_batch_size, len(shuffled_ids)) i_train_batch = shuffled_ids[start:end] if dim == 'combi': for i in i_train_batch: x_2d, x_1d = process_wav_file( this_train.wav_file.values[i], phase='TRAIN', dim=dim) x_batch.append(x_2d) x_batch_1d.append(x_1d) y_batch.append(this_train.label_id.values[i]) x_batch = np.array(x_batch) x_batch_1d = np.array(x_batch_1d) y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS)) yield [x_batch, x_batch_1d], y_batch else: for i in i_train_batch: x_batch.append( process_wav_file(this_train.wav_file.values[i], phase='TRAIN', dim=dim)) y_batch.append(this_train.label_id.values[i]) x_batch = np.array(x_batch) y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS)) yield x_batch, y_batch
def __init__(self, train_from_idx, train_to_idx, included_characters): # input image dimensions img_rows, img_cols = 28, 28 self.num_classes = 10 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) self.input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) self.input_shape = (img_rows, img_cols, 1) x_train = x_train[train_from_idx:train_to_idx, :, :, :] y_train = y_train[train_from_idx:train_to_idx] if not included_characters is None: train_idx = np.isin(y_train, included_characters) test_idx = np.isin(y_test, included_characters) y_train = y_train[train_idx] x_train = x_train[train_idx] y_test = y_test[test_idx] x_test = x_test[test_idx] x_train = x_train.astype('float32') x_test = x_test.astype('float32') self.x_train = x_train / 255 self.x_test = x_test / 255 self.y_train = to_categorical(y_train, self.num_classes) self.y_test = to_categorical(y_test, self.num_classes)
def load_raw_dataset(self): # download the extract the dataset. self.download_and_extract() # load the train and test data and labels. x_train = self.read_all_images(TRAIN_DATA_PATH) y_train = self.read_labels(TRAIN_LABELS_PATH) x_test = self.read_all_images(TEST_DATA_PATH) y_test = self.read_labels(TEST_LABELS_PATH) # convert all images to floats in the range [0, 1] x_train = x_train.astype('float32') x_train = (x_train - 10) / 255.0 x_test = x_test.astype('float32') x_test = (x_test - 10) / 255.0 # convert the labels to be zero based. y_train -= 1 y_test -= 1 # convert labels to hot-one vectors. y_train_cat = to_categorical(y_train, self.num_classes) y_test_cat = to_categorical(y_test, self.num_classes) return (x_train, y_train, y_train_cat), (x_test, y_test, y_test_cat)
def train(train_X, train_y, test_X, test_y, num_of_features, classes, layer_det, id, num_epochs=30, batch_size=100): # Train a neural network # Saves a model to disk train_y = to_categorical(train_y) #converts to one hot test_y = to_categorical(test_y) model = Sequential() model.add(Dense(layer_det[0], input_dim=num_of_features, activation='relu')) model.add(layers.Dropout(0.3, noise_shape=None, seed=None)) model.add(Dense(layer_det[1], activation='relu')) model.add(layers.Dropout(0.2, noise_shape=None, seed=None)) model.add(Dense(layer_det[2], activation='relu')) model.add(layers.Dropout(0.2, noise_shape=None, seed=None)) model.add(Dense(classes, activation='softmax')) # Compile model model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) # Fit the model model.fit(train_X, train_y, epochs=num_epochs, batch_size=batch_size) score = model.evaluate(train_X, train_y, verbose=0) print("%s: %.2f%%" % (model.metrics_names[1], score[1]*100)) model.save('model'+str(id)+'.h5') # creates a HDF5 file 'my_model.h5' score = model.evaluate(test_X,test_y, verbose=0) print("model"+str(id)+"====== %s: %.2f%%" % (model.metrics_names[1], score[1]*100))
def infer_nmt(encoder_model, decoder_model, test_en_seq, en_vsize, fr_vsize): """ Infer logic :param encoder_model: keras.Model :param decoder_model: keras.Model :param test_en_seq: sequence of word ids :param en_vsize: int :param fr_vsize: int :return: """ test_fr_seq = sents2sequences(fr_tokenizer, ['sos'], fr_vsize) test_en_onehot_seq = to_categorical(test_en_seq, num_classes=en_vsize) test_fr_onehot_seq = np.expand_dims(to_categorical(test_fr_seq, num_classes=fr_vsize), 1) enc_outs, enc_fwd_state, enc_back_state = encoder_model.predict(test_en_onehot_seq) dec_state = np.concatenate([enc_fwd_state, enc_back_state], axis=-1) attention_weights = [] fr_text = '' for i in range(fr_timesteps): dec_out, attention, dec_state = decoder_model.predict( [enc_outs, dec_state, test_fr_onehot_seq]) dec_ind = np.argmax(dec_out, axis=-1)[0, 0] if dec_ind == 0: break test_fr_seq = sents2sequences(fr_tokenizer, [fr_index2word[dec_ind]], fr_vsize) test_fr_onehot_seq = np.expand_dims(to_categorical(test_fr_seq, num_classes=fr_vsize), 1) attention_weights.append((dec_ind, attention)) fr_text += fr_index2word[dec_ind] + ' ' return fr_text, attention_weights
def prepare_training_and_testing_data(): # All images have the same square size of 28×28 pixels. # the images are grayscale. -> (train_x, train_y), (test_x, test_y) = datasets.mnist.load_data() print('Train: X=%s, y=%s' % (train_x.shape, train_x.shape)) print('Test: X=%s, y=%s' % (test_x.shape, test_x.shape)) # plot first few images for i in range(9): # define subplot plt.subplot(330 + 1 + i) # plot raw pixel data plt.imshow(train_x[i], cmap=plt.get_cmap('gray')) # show the figure plt.show() # reshape dataset to set the number of color channels # (color channel = 1, because the images are grayscale) train_x = train_x.reshape((train_x.shape[0], 28, 28, 1)) test_x = test_x.reshape((test_x.shape[0], 28, 28, 1)) train_y = to_categorical(train_y) test_y = to_categorical(test_y) train_x, test_x = normalize_image_pixel(train_x), normalize_image_pixel( test_x) return (train_x, train_y), (test_x, test_y)
def load(): """ Load the Information Bottleneck harmonics dataset Returns: Returns two namedtuples, the first one containing training and the second one containing test data respectively. Both come with fields X, y and Y: - X is the data - y is class, with numbers from 0 to 1 - Y is class, but coded as a 2-dim vector with one entry set to 1 at the column index corresponding to the class """ ID = '2017_12_21_16_51_3_275766' n_classes = 2 data_file = Path('datasets/IB_data_' + str(ID) + '.npz') if data_file.is_file(): data = np.load('datasets/IB_data_' + str(ID) + '.npz') else: import_IB_data_from_mat(ID) data = np.load('datasets/IB_data_' + str(ID) + '.npz') X_train = data['X_train'] y_train = data['y_train'] X_test = data['X_test'] y_test = data['y_test'] Y_train = keras_utils.to_categorical(y_train, n_classes).astype('float32') Y_test = keras_utils.to_categorical(y_test, n_classes).astype('float32') Dataset = namedtuple('Dataset', ['X', 'Y', 'y', 'n_classes']) training = Dataset(X_train, Y_train, y_train, int(n_classes)) test = Dataset(X_test, Y_test, y_test, int(n_classes)) return training, test
def create_dataset(file_path, persons): path = file_path + "{}_{}.txt" sgn = [] lbl = [] for i in persons: for j in range(9): with open(path.format(i, j + 1), "rb") as fp: # Unpickling data = pickle.load(fp) for k in range(np.shape(data)[0]): sgn.append(data[k]) lbl.append(j) sgn = np.asarray(sgn, dtype=np.float32) lbl = np.asarray(lbl, dtype=np.int32) c = list(zip(sgn, lbl)) shuffle(c) sgn, lbl = zip(*c) sgn = np.asarray(sgn, dtype=np.float64) lbl = np.asarray(lbl, dtype=np.int64) train_signals = sgn[0:int(0.6 * len(sgn))] train_labels = lbl[0:int(0.6 * len(lbl))] val_signals = sgn[int(0.6*len(sgn)):int(0.8*len(sgn))] val_labels = lbl[int(0.6*len(lbl)):int(0.8*len(lbl))] test_signals = sgn[int(0.8*len(sgn)):] test_labels = lbl[int(0.8*len(lbl)):] train_labels = to_categorical(train_labels) val_labels = to_categorical(val_labels) test_labels = to_categorical(test_labels) return train_signals, train_labels, val_signals, val_labels, test_signals, test_labels
def __init__(self, train_from_idx, train_to_idx): # input image dimensions img_rows, img_cols = 28, 28 self.num_classes = 10 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) self.input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) self.input_shape = (img_rows, img_cols, 1) x_train = x_train[train_from_idx:train_to_idx, :, :, :] y_train = y_train[train_from_idx:train_to_idx] x_train = x_train.astype('float32') x_test = x_test.astype('float32') self.x_train = x_train / 255 self.x_test = x_test / 255 # convert class vectors to binary class matrices self.y_train = to_categorical(y_train, self.num_classes) self.y_test = to_categorical(y_test, self.num_classes)
def main(): '''MAIN''' # setup data (train_data, train_labels), (test_data, test_labels) = load_data() one_hot_train_labels = to_categorical(train_labels) one_hot_test_labels = to_categorical(test_labels) # build model model = build_model() # train the model x_val = train_data[:1000] partial_x_train = train_data[1000:] y_val = one_hot_train_labels[:1000] partial_y_train = one_hot_train_labels[1000:] history = model.fit(partial_x_train, partial_y_train, epochs=20, batch_size=512, validation_data=(x_val, y_val)) plot_loss(history) plot_accuracy(history)
def train(full_model, en_seq, sp_seq, batch_size, n_epochs=1): """ Training the model """ for ep in range(n_epochs): losses = [] for bi in range(0, en_seq.shape[0] - batch_size, batch_size): try: en_onehot_seq = to_categorical(en_seq[bi:bi + batch_size, :], num_classes=en_vsize) sp_onehot_seq = to_categorical(sp_seq[bi:bi + batch_size, :], num_classes=sp_vsize) full_model.train_on_batch( [en_onehot_seq, sp_onehot_seq[:, :-1, :]], sp_onehot_seq[:, 1:, :]) l = full_model.evaluate( [en_onehot_seq, sp_onehot_seq[:, :-1, :]], sp_onehot_seq[:, 1:, :], batch_size=batch_size) losses.append(l) #Saving Weights if bi % 12800 == 0: infer_dec_model.save_weights('decoder_weights_n.h5') infer_enc_model.save_weights('encoder_weights_n.h5') except: continue if (ep + 1) % 1 == 0: print("Loss in epoch {}: {}".format(ep + 1, np.mean(losses)))
def get_mnist_datasets(img_h, img_w, batch_s): fashion_mnist = k_ds.fashion_mnist (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() x_train, x_test = x_train / 255.0, x_test / 255.0 # Further break training data into train / validation sets (x_train, x_valid) = x_train[5000:], x_train[:5000] (y_train, y_valid) = y_train[5000:], y_train[:5000] # Reshape input data from (28, 28) to (28, 28, 1) w, h = img_w, img_h x_train = x_train.reshape(x_train.shape[0], w, h, 1) x_valid = x_valid.reshape(x_valid.shape[0], w, h, 1) x_test = x_test.reshape(x_test.shape[0], w, h, 1) # One-hot encode the labels y_train = to_categorical(y_train, 10) y_valid = to_categorical(y_valid, 10) y_test = to_categorical(y_test, 10) train_ds = Dataset.from_tensor_slices((x_train, y_train)).shuffle(batch_s).batch(batch_s).repeat() validation_ds = Dataset.from_tensor_slices((x_valid, y_valid)).shuffle(batch_s).batch(batch_s).repeat() test_ds = Dataset.from_tensor_slices((x_test, y_test)).shuffle(batch_s).batch(batch_s) ds_lengths = (len(x_train), len(x_valid)) return train_ds, validation_ds, ds_lengths
def train_optimal_action_given_future_obs( model, target_history, target_stocks, weight_path='weights/optimal_3_stocks.h5'): (X_train, y_train), (X_test, y_test) = create_optimal_imitation_dataset(target_history) nb_classes = len(target_stocks) + 1 Y_train = to_categorical(y_train, nb_classes) Y_test = to_categorical(y_test, nb_classes) continue_train = True while continue_train: model.fit(X_train, Y_train, batch_size=128, epochs=50, validation_data=(X_test, Y_test), shuffle=True) save_weights = input('Type True to save weights\n') if save_weights: model.save(weight_path) continue_train = input('True to continue train, otherwise stop\n')
def data_mnist(): # These values are specific to MNIST img_rows = 28 img_cols = 28 nb_classes = 10 # the data, shuffled and split between train and test sets (X_train, y_train), (X_test, y_test) = mnist.load_data() if keras.backend.image_data_format() == 'th': X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols) else: X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1) X_train = X_train.astype('float32') X_test = X_test.astype('float32') X_train /= 255 X_test /= 255 print('X_train shape:', X_train.shape) print(X_train.shape[0], 'train samples') print(X_test.shape[0], 'test samples') # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) from sklearn.utils import shuffle X_train, Y_train = shuffle(X_train, Y_train) return X_train, Y_train, X_test, Y_test
def preprocess(self, df): time_of_day = df["time_of_day"].map(tod_dict).values day_of_week = df["day_of_week"].map(dow_dict).values label = df["target"].map(target2idx).values play_song = df['play_song'].map(self._to_id).values save = df['save'].map(self._to_id).values # one hot for categorical features ohe_label = to_categorical(label, num_classes=TARGET_CLASSES) ohe_tod = to_categorical(time_of_day, num_classes=TOTAL_TOD_BINS) ohe_dow = to_categorical(day_of_week, num_classes=TOTAL_DOW_BINS) # padding play_song_pad = pad_sequences(play_song, maxlen=SEQ_LEN, padding='pre', truncating='pre') save_pad = pad_sequences(save, maxlen=SEQ_LEN, padding='pre', truncating='pre') X = [play_song_pad, save_pad, ohe_tod, ohe_dow] y = ohe_label return X, y
def _prepare(self, path): self.mapping = self._load_labels_mapping() self.number_of_classes = len(self.mapping) data = loadmat(path / "emnist-byclass.mat") # load training dataset x_train = data["dataset"][0][0][0][0][0][0].astype(np.float32) x_train = x_train.reshape(x_train.shape[0], 28, 28, 1, order="A") y_train = data["dataset"][0][0][0][0][0][1] # One hot encoding y_train = to_categorical(y_train, self.number_of_classes).astype(np.int) # Calculate mean and standard deviation for input normalization self.mean = x_train.mean()#.astype(np.float32) self.std = x_train.std()#.astype(np.float32) print("Balancing train dataset...") x_train, y_train = self._sample_to_balance(x_train, y_train) # load test dataset x_test = data["dataset"][0][0][1][0][0][0].astype(np.float32) x_test = x_test.reshape(x_test.shape[0], 28, 28, 1, order="A") y_test = data["dataset"][0][0][1][0][0][1] y_test = to_categorical(y_test, self.number_of_classes).astype(np.int) # https://www.tensorflow.org/guide/datasets self.train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)) self.test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test)) print(f"Dataset ready, with {len(y_train)} training entries and {len(y_test)} test entries")
def onehot(data_tuple): """ Takes a tuple of labels as input (train, test) or (train, validation, test) and returns the one hot encoded labels for train and validation in the same tuple format. The test set is not one hot encoded. Can be chained with test_val_rest_split() or test_rest_split() """ if len(data_tuple) == 3: return ( np_utils.to_categorical(data_tuple[0]), np_utils.to_categorical(data_tuple[1]), data_tuple[2] ) elif len(data_tuple) == 2: return ( np_utils.to_categorical(data_tuple[0]), data_tuple[1] ) elif len(data_tuple) == 1: return ( data_tuple[0] ) else: raise TypeError('Wrong number of arguments')
def train(full_model, infer_enc_model, infer_dec_model, en_seq, fr_seq): """ Training the model """ for ep in range(N_EPOCHS): losses = [] start = time.time() for bi in tqdm(range(0, en_seq.shape[0] - BATCH_SIZE, BATCH_SIZE)): en_onehot_seq = to_categorical(en_seq[bi:bi + BATCH_SIZE, :], num_classes=en_vsize) fr_onehot_seq = to_categorical(fr_seq[bi:bi + BATCH_SIZE, :], num_classes=fr_vsize) full_model.train_on_batch( [en_onehot_seq, fr_onehot_seq[:, :-1, :]], fr_onehot_seq[:, 1:, :]) l = full_model.evaluate([en_onehot_seq, fr_onehot_seq[:, :-1, :]], fr_onehot_seq[:, 1:, :], batch_size=BATCH_SIZE, verbose=0) losses.append(l) end = time.time() # if (ep + 1) % 5 == 0: # save model every 5 epochs # save_model(full_model, ep+1) # show test results after epoch test_inferring(infer_enc_model, infer_dec_model) logger.info("Elapsed: {} sec. Loss in epoch {}/{}: {}".format( round(end - start, 3), ep + 1, N_EPOCHS, np.mean(losses)))
def load(): """Load the MNIST handwritten digits dataset Returns: Returns two namedtuples, the first one containing training and the second one containing test data respectively. Both come with fields X, y and Y: - X is the data - y is class, with numbers from 0 to 9 - Y is class, but coded as a 10-dim vector with one entry set to 1 at the column index corresponding to the class """ n_classes = 10 (X_train, y_train), (X_test, y_test) = keras.datasets.mnist.load_data() X_train = np.reshape(X_train, [X_train.shape[0], -1]).astype('float32') / 255.0 X_test = np.reshape(X_test, [X_test.shape[0], -1]).astype('float32') / 255.0 X_train = X_train * 2.0 - 1.0 X_test = X_test * 2.0 - 1.0 Y_train = keras_utils.to_categorical(y_train, n_classes).astype('float32') Y_test = keras_utils.to_categorical(y_test, n_classes).astype('float32') Dataset = namedtuple('Dataset', ['X', 'Y', 'y', 'n_classes']) training = Dataset(X_train, Y_train, y_train, n_classes) test = Dataset(X_test, Y_test, y_test, n_classes) return training, test
def main(): (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() x_train = x_train.reshape(60000, 784) x_train = x_train / 255 x_test = x_test.reshape(10000, 784) x_test = x_test / 255 y_train = utils.to_categorical(y_train, 10) y_test = utils.to_categorical(y_test, 10) model = Sequential() model.add(Dense(784, input_dim=784, activation="relu")) model.add(Dense(10, activation="softmax")) model.compile(loss="categorical_crossentropy", optimizer="SGD", metrics=["accuracy"]) model.summary() callback = [ TensorBoard(log_dir='logs', histogram_freq=1, write_images=True) ] model.fit(x_train, y_train, batch_size=200, epochs=300, verbose=1, validation_split=0.2, callbacks=callback) model.save("fashion_model.h5") score = model.evaluate(x_test, y_test, verbose=1) print("Accuracy on test data is", score[1] * 100, "percent")
def infer_nmt(encoder_model, decoder_model, test_en_seq, en_vsize, sp_vsize): """ Infer logic :param encoder_model: keras.Model :param decoder_model: keras.Model :param test_en_seq: sequence of word ids :param en_vsize: int :param sp_vsize: int :return: """ test_sp_seq = sents2sequences(sp_tokenizer, ['sos'], sp_vsize) test_en_onehot_seq = to_categorical(test_en_seq, num_classes=en_vsize) test_sp_onehot_seq = np.expand_dims( to_categorical(test_sp_seq, num_classes=sp_vsize), 1) enc_outs, enc_last_state = encoder_model.predict(test_en_onehot_seq) dec_state = enc_last_state attention_weights = [] sp_text = '' for i in range(20): dec_out, attention, dec_state = decoder_model.predict( [enc_outs, dec_state, test_sp_onehot_seq]) dec_ind = np.argmax(dec_out, axis=-1)[0, 0] #print('Decoder Output Top 10', dec_out[0,0,:10]) if dec_ind == 0: break test_sp_seq = sents2sequences(sp_tokenizer, [sp_tokenizer.index_word[dec_ind]], sp_vsize) test_sp_onehot_seq = np.expand_dims( to_categorical(test_sp_seq, num_classes=sp_vsize), 1) attention_weights.append((dec_ind, attention)) sp_text += sp_tokenizer.index_word[dec_ind] + ' ' return sp_text, attention_weights
def main(): (x_train, y_train), (x_test, y_test) = cifar10.load_data() y_train = to_categorical(y_train) y_test = to_categorical(y_test) model = ResNet50(input_shape=(32, 32, 3), include_top=False, weights='imagenet', classes=10) model.summary()
def load_mnist(): (train_x, train_y), (test_x, test_y) = mnist.load_data() train_x = train_x.reshape(train_x.shape[0], 784).astype('float32') / 255.0 test_x = test_x.reshape(test_x.shape[0], 784).astype('float32') / 255.0 train_y = to_categorical(train_y, 10) test_y = to_categorical(test_y, 10) return (train_x, train_y), (test_x, test_y)
def load_data(): ''' load data from MovieLens 100K Dataset http://grouplens.org/datasets/movielens/ Note that this method uses ua.base and ua.test in the dataset. :return: train_users, train_x, test_users, test_x :rtype: list of int, numpy.array, list of int, numpy.array ''' path = get_file( 'ml-100k.zip', origin='http://files.grouplens.org/datasets/movielens/ml-100k.zip') with ZipFile(path, 'r') as ml_zip: max_item_id = -1 train_history = {} with ml_zip.open('ml-100k/ua.base', 'r') as file: for line in file: user_id, item_id, rating, timestamp = line.decode( 'utf-8').rstrip().split('\t') if int(user_id) not in train_history: train_history[int(user_id)] = [int(item_id)] else: train_history[int(user_id)].append(int(item_id)) if max_item_id < int(item_id): max_item_id = int(item_id) test_history = {} with ml_zip.open('ml-100k/ua.test', 'r') as file: for line in file: user_id, item_id, rating, timestamp = line.decode( 'utf-8').rstrip().split('\t') if int(user_id) not in test_history: test_history[int(user_id)] = [int(item_id)] else: test_history[int(user_id)].append(int(item_id)) max_item_id += 1 # item_id starts from 1 train_users = list(train_history.keys()) train_x = numpy.zeros((len(train_users), max_item_id), dtype=numpy.int32) print(train_x.shape) for i, hist in enumerate(train_history.values()): # print(hist) mat = to_categorical(hist, max_item_id) # print(mat.shape) train_x[i] = numpy.sum(mat, axis=0) # print(len(train_x[i])) test_users = list(test_history.keys()) test_x = numpy.zeros((len(test_users), max_item_id), dtype=numpy.int32) for i, hist in enumerate(test_history.values()): mat = to_categorical(hist, max_item_id) test_x[i] = numpy.sum(mat, axis=0) return train_users, train_x, test_users, test_x
def get_dataset(class_type='gender'): """@:param class_type: str - type of class needed to be in Y. values { 'gender' , 'speaker' } """ train = [] test = [] valid = [] speaker_ids = get_speaker_ids() for s in speaker_ids: file_list = glob.glob(DATA_DIR + s + '/*/*.wav') print("Loading Data from :", DATA_DIR + s) all_data = [] for f in file_list: speaker_id = f.split("/")[SPEAKER_IDX] chapter_id = f.split("/")[CHAPTER_IDX] filename = f.split("/")[FILENAME_IDX] all_data.append( os.path.join( DATA_DIR, os.path.join( speaker_id, os.path.join(chapter_id, os.path.join(filename))))) random.shuffle(all_data) split_tuple = np.split( np.array(all_data), [int(0.7 * len(all_data)), int(0.9 * len(all_data))]) train = train + split_tuple[0].tolist() test = test + split_tuple[1].tolist() valid = valid + split_tuple[2].tolist() if class_type == 'gender': x_train, y_train = get_XY_gender(train) x_test, y_test = get_XY_gender(test) x_valid, y_valid = get_XY_gender(valid) num_classes = len(GENDER_CLASSES) elif class_type == 'speaker': x_train, y_train = get_XY_speaker(train) x_test, y_test = get_XY_speaker(test) x_valid, y_valid = get_XY_speaker(valid) num_classes = NUM_CLASSES else: print("Invalid class_type. Required 'gender' or 'speaker'. Given: {}". format(class_type)) return return (x_train, to_categorical(y_train, num_classes=num_classes)), \ (x_test, to_categorical(y_test, num_classes=num_classes)), \ (x_valid, to_categorical(y_valid, num_classes=num_classes))
def get_input_datasets(use_bfloat16=False): """Downloads the MNIST dataset and creates train and eval dataset objects. Args: use_bfloat16: Boolean to determine if input should be cast to bfloat16 Returns: Train dataset and eval dataset. The dataset doesn't include batch dim. """ cast_dtype = dtypes.bfloat16 if use_bfloat16 else dtypes.float32 # the data, split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() train_data_shape = (x_train.shape[0],) + get_data_shape() test_data_shape = (x_test.shape[0],) + get_data_shape() if backend.image_data_format() == 'channels_first': x_train = x_train.reshape(train_data_shape) x_test = x_test.reshape(test_data_shape) else: x_train = x_train.reshape(train_data_shape) x_test = x_test.reshape(test_data_shape) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 # convert class vectors to binary class matrices y_train = utils.to_categorical(y_train, NUM_CLASSES) y_test = utils.to_categorical(y_test, NUM_CLASSES) # train dataset train_ds = dataset_ops.Dataset.from_tensor_slices((x_train, y_train)) # TODO(rchao): Remove maybe_shard_dataset() once auto-sharding is done. train_ds = maybe_shard_dataset(train_ds) train_ds = train_ds.repeat() train_ds = train_ds.map(lambda x, y: (math_ops.cast(x, cast_dtype), y)) train_ds = train_ds.batch(64, drop_remainder=True) # eval dataset eval_ds = dataset_ops.Dataset.from_tensor_slices((x_test, y_test)) # TODO(rchao): Remove maybe_shard_dataset() once auto-sharding is done. eval_ds = maybe_shard_dataset(eval_ds) eval_ds = eval_ds.repeat() eval_ds = eval_ds.map(lambda x, y: (math_ops.cast(x, cast_dtype), y)) eval_ds = eval_ds.batch(64, drop_remainder=True) return train_ds, eval_ds
def encode_x(self, x): idx_x = [self.s_to_idx[s] for s in x] return to_categorical(idx_x, num_classes=self.n_symbols)
def encode_y(self, y): idx_y = self.c_to_idx[y] return to_categorical(idx_y, num_classes=self.n_classes)