def main(): """ メイン実行関数 """ X_train, X_test, y_train, y_test = np.load('./animal_aug.npy', allow_pickle=True) X_train = X_train.astype('float') / 256 X_test = X_test.astype('float') / 256 y_train = np_utils.to_categorical(y_train, num_classes) y_test = np_utils.to_categorical(y_test, num_classes) model = model_train(X_train, y_train) model_eval(model, X_test, y_test)
def get_data(): # the data, shuffled and split between tran and test sets (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train = X_train.reshape(60000, 784)[:max_train_samples] X_test = X_test.reshape(10000, 784)[:max_test_samples] X_train = X_train.astype('float32') / 255 X_test = X_test.astype('float32') / 255 # convert class vectors to binary class matrices y_train = y_train[:max_train_samples] y_test = y_test[:max_test_samples] Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) test_ids = np.where(y_test == np.array(weighted_class))[0] return (X_train, Y_train), (X_test, Y_test), test_ids
def prepare_sequences(notes, n_vocab): """ Prepare the sequences used by the Neural Network """ sequence_length = 100 # get all pitch names pitchnames = sorted(set(item for item in notes)) # create a dictionary to map pitches to integers note_to_int = dict((note, number) for number, note in enumerate(pitchnames)) network_input = [] network_output = [] # create input sequences and the corresponding outputs for i in range(0, len(notes) - sequence_length, 1): sequence_in = notes[i:i + sequence_length] sequence_out = notes[i + sequence_length] network_input.append([note_to_int[char] for char in sequence_in]) network_output.append(note_to_int[sequence_out]) n_patterns = len(network_input) # reshape the input into a format compatible with LSTM layers network_input = numpy.reshape(network_input, (n_patterns, sequence_length, 1)) # normalize input network_input = network_input / float(n_vocab) network_output = np_utils.to_categorical(network_output) return (network_input, network_output)
def generateData(batch_size, data=[]): #print 'generateData...' while True: train_data = [] train_label = [] batch = 0 for i in (range(len(data))): url = data[i] batch += 1 img = load_img(filepath + 'src/' + url) img = img_to_array(img) train_data.append(img) label = load_img(filepath + 'label/' + url, grayscale=True) label = img_to_array(label).reshape((img_w * img_h, )) # print label.shape train_label.append(label) if batch % batch_size == 0: #print 'get enough bacth!\n' train_data = np.array(train_data) train_label = np.array(train_label).flatten() train_label = labelencoder.transform(train_label) train_label = to_categorical(train_label, num_classes=n_label) train_label = train_label.reshape( (batch_size, img_w * img_h, n_label)) yield (train_data, train_label) train_data = [] train_label = [] batch = 0
def generateValidData(batch_size, data=[]): #print 'generateValidData...' while True: valid_data = [] valid_label = [] batch = 0 for i in (range(len(data))): url = data[i] batch += 1 img = load_img(filepath + 'src/' + url) img = img_to_array(img) valid_data.append(img) label = load_img(filepath + 'label/' + url, grayscale=True) label = img_to_array(label).reshape((img_w * img_h, )) # print label.shape valid_label.append(label) if batch % batch_size == 0: valid_data = np.array(valid_data) valid_label = np.array(valid_label).flatten() valid_label = labelencoder.transform(valid_label) valid_label = to_categorical(valid_label, num_classes=n_label) valid_label = valid_label.reshape( (batch_size, img_w * img_h, n_label)) yield (valid_data, valid_label) valid_data = [] valid_label = [] batch = 0
def bd(path,optimizer='SGD',loss='categorical_crossentropy',metrics=["accuracy"],*args): def glob(path): from glob import glob any = glob(path) return any def shaping(files):#get files from cv2 import imread from numpy import array imgs,av = [],[] for path in files: p = path_name+path[-1]+'/*.jpg' p = glob(p) for f in p: av.append(int(path[-1])) image = imread(f) imgs.append(image) av = array(av,dtype='uint8') imgs = array(imgs) return imgs,av from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPooling2D from tensorflow.keras.utils import np_utils #setting types files = glob(path+'*') x_train, y_train = shaping(files) f = [len(x_train[0]),len(x_train[0,0]),len(x_train[0,0,0])] x_train = x_train.astype('float64')/255 y_train = np_utils.to_categorical(y_train, len(files)).astype('uint8') #test unit(validation) x_test = x_train y_test = y_train #pr-process model = Sequential() model.add(Conv2D(16, (4,3), input_shape=(f[0],f[1],f[2]))) model.add(Conv2D(16, (4,3))) model.add(MaxPooling2D(pool_size=(2,2))) model.add(Dropout(0.25)) model.add(Conv2D(16, (4,3))) model.add(Conv2D(16, (4,3))) model.add(MaxPooling2D(pool_size=(2,2))) model.add(Dropout(0.25)) model.add(Flatten()) #dense model.add(Dense(units=32,activation="tanh",name="dense_1")) model.add(Dense(units=16,activation="tanh",name="dense_2")) model.add(Dense(units=8,activation="tanh",name="dense_3")) model.add(Dense(units=len(files),activation= 'softmax' if len(files)>2 else 'sigmoid',name="dense_f")) try: model.load_weights("my_weight.h5") except OSError: print("weights do not exist") #compile model.compile(optimizer=optimizer, loss=loss, metrics=metrics) return model,x_test,x_train,y_test,y_train
def train(): with open("train_features", "rb") as f: train_images = np.array(pickle.load(f)) with open("train_labels", "rb") as f: train_labels = np.array(pickle.load(f), dtype=np.int32) with open("test_features", "rb") as f: test_images = np.array(pickle.load(f)) with open("test_labels", "rb") as f: test_labels = np.array(pickle.load(f), dtype=np.int32) train_labels = np_utils.to_categorical(train_labels) test_labels = np_utils.to_categorical(test_labels) model, callbacks_list = mlp_model() model.fit(train_images, train_labels, validation_data=( test_images, test_labels), epochs=1500, batch_size=50, callbacks=callbacks_list) scores = model.evaluate(test_images, test_labels, verbose=3) print(scores) print("MLP Error: %.2f%%" % (100-scores[1]*100))
def sample_to_x_y_bq_worker(sample, max_label_len, label_encoding, sparse_labels, n_classes): """Convert a `medaka.common.Sample` object into an x,y tuple for training. :param sample: (filename, sample key) :param max_label_len: int, maximum label length, longer labels will be truncated. :param label_encoding: {label: int encoded label}. :param sparse_labels: bool, create sparse labels. :param n_classes: int, number of label classes. :returns: (np.ndarray of inputs, np.ndarray of labels) """ sample_key, sample_file = sample with medaka.datastore.DataStore(sample_file) as ds: s = ds.load_sample(sample_key) if s.labels is None: raise ValueError("Sample {} in {} has no labels.".format( sample_key, sample_file)) x = s.features # s.labels is a structured array with run-length encoded (base, length) labels. # the dimension of the last axis determines the ploidy. ploidy = s.labels.shape[-1] # trim label lengths to max_label_len s.labels['run_length'] = np.minimum(s.labels['run_length'], max_label_len, out=s.labels['run_length']) hap_ys = [] for p in range(ploidy): hap_labels = s.labels[:, p] hap_ys.append( np.fromiter((label_encoding[tuple(l)] for l in hap_labels), dtype=int, count=len(hap_labels))) if ploidy == 1: y = hap_ys[0].reshape(hap_ys[0].shape + (1, )) if not sparse_labels: from tensorflow.keras.utils.np_utils import to_categorical y = to_categorical(y, num_classes=n_classes) elif not sparse_labels: # multi-hot-encoding, heterozygous loci have >1 non-zero elements y = np.zeros(shape=(len(s.labels), len(label_encoding)), dtype=int) for hap_y in hap_ys: np.put_along_axis(y, hap_y.reshape(-1, 1), 1, axis=1) else: #TODO one could implement a sparse labeling scheme encoding pairs of labels # either in a phased or unphased manner raise NotImplementedError( 'Training with ploidy >1 and sparse labels is not implemented.' ) return x, y
def load_face_data(): data_path = "7-2P-dataset" if not os.path.isdir(data_path): download_url(output_path='dataset.zip', url=urls[data_path]) categories = os.listdir(data_path) labels = [i for i in range(len(categories))] label_dict = dict(zip(categories, labels)) # empty dictionary img_size = 100 data = [] target = [] for category in categories: folder_path = os.path.join(data_path, category) img_names = os.listdir(folder_path) for img_name in img_names: img_path = os.path.join(folder_path, img_name) img = cv2.imread(img_path) try: gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # Converting the image into gray scale resized = cv2.resize(gray, (img_size, img_size)) # resizing the gray scale into 50x50, since we need a fixed common size for all the images in the dataset data.append(resized) target.append(label_dict[category]) # appending the image and the label(categorized) into the list (dataset) except Exception as e: print('Exception:', e) # if any exception raised, the exception will be printed here. And pass to the next image data = np.array(data) / 1.0 data = np.reshape(data, (data.shape[0], img_size, img_size, 1)) target = np.array(target) new_target = to_categorical(target) train_data, test_data, train_target, test_target = train_test_split( data, new_target, test_size=0.2, stratify=new_target) return train_data, test_data, train_target, test_target
def next(self): """Next batch.""" with self.lock: index_array, current_index, current_batch_size = next( self.index_generator) batch_x = np.zeros( (current_batch_size,) + self.image_shape, dtype=K.floatx()) batch_y = np.zeros( (current_batch_size,) + self.label_shape, dtype=np.int8) #batch_y = np.reshape(batch_y, (current_batch_size, -1, self.classes)) for i, j in enumerate(index_array): fn = self.filenames[j] x = self.image_set_loader.load_img(fn) x = self.image_data_generator.standardize(x) batch_x[i] = x y = self.image_set_loader.load_seg(fn) y = to_categorical(y, self.classes).reshape(self.label_shape) #y = np.reshape(y, (-1, self.classes)) batch_y[i] = y # save augmented images to disk for debugging #if self.image_set_loader.save_to_dir: # for i in range(current_batch_size): # x = batch_x[i] # y = batch_y[i].argmax( # self.image_data_generator.channel_axis - 1) # if self.image_data_generator.data_format == 'channels_first': # y = y[np.newaxis, ...] # else: # y = y[..., np.newaxis] # self.image_set_loader.save(x, y, current_index + i) return batch_x, batch_y
seq_length = 100 dataX = [] dataY = [] for i in range(0, n_chars - seq_length, 1): seq_in = raw_text[i:i + seq_length] seq_out = raw_text[i + seq_length] dataX.append([char_to_int[char] for char in seq_in]) dataY.append(char_to_int[seq_out]) n_patterns = len(dataX) print("Total Patterns: ", n_patterns) # reshape X to be [samples, time steps, features] X = numpy.reshape(dataX, (n_patterns, seq_length, 1)) # normalize X = X / float(n_vocab) # one hot encode the output variable y = np_utils.to_categorical(dataY) # define the LSTM model model = Sequential() model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True)) model.add(Dropout(0.2)) model.add(LSTM(256)) model.add(Dropout(0.2)) model.add(Dense(y.shape[1], activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam') # define the checkpoint filepath = "F:\\PARAM\\Wiki-Movies\\weights-improvement-{epoch:02d}-{loss:.4f}-bigger.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min') callbacks_list = [checkpoint] # fit the model model.fit(X, y, epochs=50, batch_size=64, callbacks=callbacks_list)
from keras_contrib.applications.wide_resnet import WideResidualNetwork batch_size = 64 epochs = 300 img_rows, img_cols = 32, 32 (trainX, trainY), (testX, testY) = cifar10.load_data() trainX = trainX.astype('float32') trainX /= 255.0 testX = testX.astype('float32') testX /= 255.0 tempY = testY trainY = kutils.to_categorical(trainY) testY = kutils.to_categorical(testY) generator = ImageDataGenerator(rotation_range=10, width_shift_range=5. / 32, height_shift_range=5. / 32, horizontal_flip=True) generator.fit(trainX, seed=0, augment=True) # We will be training the model, therefore no need to load weights model = WideResidualNetwork(depth=28, width=8, dropout_rate=0.0, weights=None) model.summary() model.compile(loss='categorical_crossentropy',
def main(): class_names = ["English", "Swedish", "Spanish", "Portuguese", "Russian"] all_data = read_all(class_names) train, validation, test = split_data(all_data) #print("Printing first 50 training tweets and its labels before making changes of input representation:") #print(train[:50]) test_lines = test x_train = np.asarray([np.asarray(text) for text in train['tweets']]) y_train = np.asarray([np.asarray(label) for label in train['language']]) x_validation = np.asarray( [np.asarray(text) for text in validation['tweets']]) y_validation = np.asarray( [np.asarray(label) for label in validation['language']]) x_test = np.asarray([np.asarray(text) for text in test['tweets']]) y_test = np.asarray([np.asarray(label) for label in test['language']]) x_train = [encode_ngram_text(line, 1) for line in x_train] x_train = pad(x_train, max_length) for train in x_train[:3]: print(train) x_validation = [encode_ngram_text(line, 1) for line in x_validation] x_validation = pad(x_validation, max_length) x_test = [encode_ngram_text(line, 1) for line in x_test] x_test = pad(x_test, max_length) x_train = np.asarray(x_train) x_validation = np.asarray(x_validation) x_test = np.asarray(x_test) y_train = np.asarray(y_train) y_validation = np.asarray(y_validation) y_test = np.asarray(y_test) histo_train = np.histogram(y_train, bins=5) histo_vali = np.histogram(y_validation, bins=5) histo_test = np.histogram(y_test, bins=5) print("Training histogram", histo_train) print("Validation histogram", histo_vali) print("Test histogram", histo_test) y_train = np_utils.to_categorical(y_train, num_classes=5) y_validation = np_utils.to_categorical(y_validation, num_classes=5) y_test = np_utils.to_categorical(y_test, num_classes=5) print("Length of x_train:", len(x_train)) print("Length of x_validation:", len(x_validation)) print("Length of x_test:", len(x_test)) x_train = x_train.reshape(4001, 1, 150) print("Shape of x_train:", x_train.shape) x_validation = x_validation.reshape(500, 1, 150) print("Shape of x_validate:", x_validation.shape) x_test = x_test.reshape(501, 1, 150) print("Shape of x_test:", x_test.shape) print("Shape of y_train:", y_train.shape) print("Shape of y_validate:", y_validation.shape) print("Shape of y_test:", y_test.shape) print( "Printing 3 first training tweets and its labels as final input form:") print(x_train[:3]) print(y_train[:3]) print('Build model...') model = Sequential() model.add(LSTM(64, input_shape=(1, 150))) model.add(Dropout(0.5)) model.add(Dense(units=len(class_names))) model.add(Activation('softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) history = model.fit(x_train, y_train, batch_size=10, epochs=10, validation_data=(x_validation, y_validation), verbose=1) loss, acc = model.evaluate(x_validation, y_validation, verbose=1) print("Loss: %.2f" % (loss)) print("Validation Accuracy: %.2f" % (acc)) #print("Test (hold out data set) Accuracy: %.2f" % (acc)) loss2, acc3 = model.evaluate(x_test, y_test, verbose=1) print("Loss: %.2f" % (loss2)) print("Test (hold-out-dataset) Accuracy: %.2f" % (acc3)) print('\n# Generate predictions for 3 samples') print(test_lines) predictions = model.predict(x_test) sum = 0 for i in range(len(x_test)): new_pred = np.argmax(predictions[i]) old_pred = np.argmax(y_test[i]) print('prediction:', new_pred, "Correct label:", old_pred) if new_pred == old_pred: sum += 1 print(sum / len(x_test)) # Plot training & validation accuracy values plt.plot(history.history['acc']) plt.plot(history.history['val_acc']) #plt.axhline(y=0.85, color='r', linestyle='--') plt.title('Model accuracy') plt.ylabel('Accuracy') plt.xlabel('Epoch') plt.legend(['Train', 'Val'], loc='upper left') plt.savefig('model_3_test_fig.png')
batch_size = 128 nb_classes = 10 nb_epoch = 600 data_augmentation = True # input image dimensions img_rows, img_cols = 32, 32 # The CIFAR10 images are RGB. img_channels = 3 # The data, shuffled and split between train and test sets: (X_train, y_train), (X_test, y_test) = cifar10.load_data() # Convert class vectors to binary class matrices. Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) X_train = X_train.astype('float32') X_test = X_test.astype('float32') # preprocess input X_train = preprocess_input(X_train) X_test = preprocess_input(X_test) # For training, the auxilary branch must be used to correctly train NASNet model = NASNetCIFAR((img_rows, img_cols, img_channels), use_auxilary_branch=True) model.summary() optimizer = Adam(lr=1e-3, clipnorm=5)
optimizer = Adam(lr=1e-3) # Using Adam instead of SGD to speed up training model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc']) print('Finished compiling') (trainX, trainY), (testX, testY) = cifar10.load_data() trainX = trainX.astype('float32') testX = testX.astype('float32') trainX /= 255. testX /= 255. Y_train = np_utils.to_categorical(trainY, nb_classes) Y_test = np_utils.to_categorical(testY, nb_classes) generator = ImageDataGenerator(rotation_range=15, width_shift_range=5. / 32, height_shift_range=5. / 32) generator.fit(trainX, seed=0) weights_file = 'DenseNet-40-12-CIFAR-10.h5' lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=np.sqrt(0.1), cooldown=0, patience=10, min_lr=0.5e-6)
from tensorflow.keras.datasets import mnist from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Activation, Conv2D, MaxPooling2D, Flatten, Dropout from tensorflow.keras.utils import np_utils input_size = 784 hidden_neurons = 200 classes = 10 input_shape = (28, 28, 1) (X_train, Y_train), (X_test, Y_test) = mnist.load_data() x_train = X_train.reshape(60000, 28, 28, 1)/255 x_test = X_test.reshape(10000, 28, 28, 1)/255 y_train = np_utils.to_categorical(Y_train) y_test = np_utils.to_categorical(Y_test) model = Sequential() model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1))) model.add(MaxPooling2D((2, 2))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D((2, 2))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(256, activation='relu')) model.add(Dense(10, activation='softmax')) model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')