def main(): args = parse_option() print(args) x_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train') x_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k') class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot'] x_train=x_train.reshape(x_train.shape[0], 28, 28, 1) x_test=x_test.reshape(x_test.shape[0], 28, 28 ,1) x_train = x_train / 255.0 x_test = x_test / 255.0 y_train=keras.utils.to_categorical(y_train) y_test=keras.utils.to_categorical(y_test) num_classes = 10 print("Grid search for batch_size,batch norm and learning rate") model = KerasClassifier(build_fn=build_model,,epochs=40,verbose=1) batch_size = [32,64,128] lr = [0.01,0.001] use_bn = [True,False] param_grid = dict(batch_size=batch_size, lr=lr,use_bn=use_bn) grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3) grid_result = grid.fit(x_train, y_train) print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) means = grid_result.cv_results_['mean_test_score'] stds = grid_result.cv_results_['std_test_score'] params = grid_result.cv_results_['params'] for mean, stdev, param in zip(means, stds, params): print("%f (%f) with: %r" % (mean, stdev, param))
def load_data(): print("Loading data..") X_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train') X_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k') # we want to work on a binary classification problem so only class 5 and class 7 will be used tr_mask = np.where(np.logical_or(y_train == 5, y_train == 7))[0] te_mask = np.where(np.logical_or(y_test == 5, y_test == 7))[0] X_train = X_train[tr_mask, :] y_train = y_train[tr_mask] X_test = X_test[te_mask, :] y_test = y_test[te_mask] # let class 5 be 0 and let class 7 be 1 y_train[y_train == 5] = 0 y_train[y_train == 7] = 1 y_test[y_test == 5] = 0 y_test[y_test == 7] = 1 # Debugging assert X_train.shape[0] == y_train.shape[0] assert X_test.shape[0] == y_test.shape[0] # plt.imshow(X_train[1251].reshape(28,28),cmap=plt.cm.gray) # plt.show() print("Done..") return X_train, y_train, X_test, y_test
def __init__(self, part='train', batch_size=128, partial=1000): from utils.mnist_reader import load_mnist train_raw, _ = load_mnist('datasets/fashion', kind='train') test_raw, _ = load_mnist('datasets/fashion', kind='t10k') grey_scale = lambda x: np.float32(x / 255.) def static_binarize(d): ids = d < 0.5 d[ids] = 0. d[~ids] = 1. train_data = grey_scale(train_raw) test_data = grey_scale(test_raw) static_binarize(train_data) static_binarize(test_data) assert train_data.shape == (60000, 784) assert test_data.shape == (10000, 784) self.data = { 'train': train_data[:55000], 'valid': train_data[55000:], 'test': test_data, 'partial_train': train_data[:partial], 'partial_test': test_data[:partial], }[part] self.size = self.data.shape[0] self.batch_size = batch_size self._construct()
def __init__(self, pending_q: Queue) -> None: super().__init__() self.pending_q = pending_q X, self.Y = mnist_reader.load_mnist(path=DATA_DIR, kind='train') Xt, self.Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k') scaler = preprocessing.StandardScaler().fit(X) self.X = scaler.transform(X) self.Xt = scaler.transform(Xt)
def __init__(self, pending_q: Queue) -> None: super().__init__() self.pending_q = pending_q X, self.Y = mnist_reader.load_mnist(path=DATA_DIR, kind='train') Xt, self.Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k') scaler = preprocessing.StandardScaler().fit(X.astype(float)) self.X = scaler.transform(X.astype(float)) self.Xt = scaler.transform(Xt.astype(float))
def init(): x_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train') x_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k') x_train = np.array(x_train) y_train = np.array(y_train) x_test = np.array(x_test) y_test = np.array(y_test) x_train_temp = [] y_train_temp = [] x_test_temp = [] y_test_temp = [] sandal_count = 0 sneaker_count = 0 train_len = len(x_train) test_len = len(x_test) # Convert the training data into a binary classification problem. for i in range(train_len): # Let sandal data be class 0. if y_train[i] == 5 and sandal_count < 3000: y_train_temp.append(0) x_train_temp.append(x_train[i]) sandal_count = sandal_count + 1 # Let sneaker data be class 1. if y_train[i] == 7 and sneaker_count < 3000: y_train_temp.append(1) x_train_temp.append(x_train[i]) sneaker_count = sneaker_count + 1 # Convert the test data into a binary classification problem. for i in range(test_len): # Let sandal data be class 0. if y_test[i] == 5: y_test_temp.append(0) x_test_temp.append(x_test[i]) # Let sneaker data be class 1. if y_test[i] == 7: y_test_temp.append(1) x_test_temp.append(x_test[i]) x_train = np.array(x_train_temp) / 255 y_train = y_train_temp x_test = np.array(x_test_temp) / 255 y_test = y_test_temp print("Training set size: " + str(len(x_train))) print("Test set size: " + str(len(x_test))) return x_train, y_train, x_test, y_test
def main(): x_train, y_train = load_mnist('data/fashion', kind='train') x_test, y_test = load_mnist('data/fashion', kind='t10k') x_train, y_train = shuffle(x_train, y_train) x_test, y_test = shuffle(x_test, y_test) if LESS_DATA: x_train, y_train = x_train[0:10000, ], y_train[0:10000, ] x_test, y_test = x_test[0:1000, ], y_train[0:1000, ] num_classes = len(set(y_train)) # One-hot encoding y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) input_img = Input(shape=(28, 28, 1)) x = Conv2D(32, kernel_size=(3, 3), activation='relu')(input_img) x = Conv2D(64, (3, 3), activation='relu')(x) x = MaxPooling2D(pool_size=(2, 2))(x) x = Dropout(0.25)(x) x = Flatten()(x) x = Dense(128, activation='relu')(x) x = Dropout(0.5)(x) output = Dense(num_classes, activation='softmax')(x) model = Model(inputs=input_img, outputs=output) model.summary() model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1, validation_data=(x_test, y_test)) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) # convert keras model to pb files output_names = [out.op.name for out in model.outputs] frozen_graph = utils.freeze_session(keras.backend.get_session(), output_names=output_names) tf.train.write_graph(frozen_graph, "output_models/", "keras_fashion.pb", as_text=False) tf.train.write_graph(frozen_graph, "output_models/", "keras_fashion.pbtxt", as_text=True)
def _load_data(self): X_train, y_train = load_mnist('./../../submodules/fashion-mnist/data/fashion', kind='train') X_test, y_test = load_mnist('./../../submodules/fashion-mnist/data/fashion/', kind='t10k') train_data = X_train train_target = y_train test_data = X_test test_target = y_test return train_data, train_target, test_data, test_target
def load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: X_train, y_train = load_mnist('./../../submodules/fashion-mnist/data/fashion', kind='train') X_test, y_test = load_mnist('./../../submodules/fashion-mnist/data/fashion/', kind='t10k') train_data = self._normalise(X_train) train_target = y_train test_data = self._normalise(X_test) test_target = y_test return train_data, train_target, test_data, test_target
def euclidean_tests(): X_train, y_train = mnist_reader.load_mnist('../data/fashion', kind='train') X_test, y_test = mnist_reader.load_mnist('../data/fashion', kind='t10k') X_train, X_test = X_train.astype('float32') / 255, X_test.astype( 'float32') / 255 k_values = [3, 4, 5, 6, 7, 8, 9] print("StartTime =", datetime.now().strftime("%H:%M:%S")) error_best, best_k, errors = model_selection_knn_euclidean( X_test, X_train, y_test, y_train, k_values) print("EndTime =", datetime.now().strftime("%H:%M:%S")) print("error: ", error_best, "best_k: ", best_k)
def import_data(): X_train_mnist, y_train_mnist = mnist_reader.load_mnist('data/fashion', kind='train') X_test_mnist, y_test_mnist = mnist_reader.load_mnist('data/fashion', kind='t10k') for i in range(y_train_mnist.size): if y_train_mnist[i] < 5: y_train_.append(y_train_mnist[i]) x_train_.append(X_train_mnist[i]) return np.array(x_train_), np.array(y_train_)
def readfile(dir): train_img, train_label = mnist_reader.load_mnist(path=dir, kind='train') test_img, test_label = mnist_reader.load_mnist(path=dir, kind='t10k') mu=np.mean(train_img, axis=0, dtype=np.float64) x=train_img-mu cov=np.dot(np.transpose(x), x) cov=np.divide(cov, len(x)) [egval, egvec]=la.eig(cov) idx=np.argsort(egval) egval=egval[idx] egvec=egvec[:, idx] return [egvec, train_img, test_img, train_label, test_label]
def prepare_loaders(config, logger, skip_tst_dataset, multiply_by_number_of_gpu=False): from radim.FashionMnistDataset import FashionMnistDataset batch_size = int(config['trn']['batch_size']) if multiply_by_number_of_gpu: batch_size = torch.cuda.device_count() * int( config['trn']['batch_size']) logger.info(" Batch size: %d" % batch_size) from utils.mnist_reader import load_mnist X, y = load_mnist(config['trn']['dataset_dir'], kind='train') X_tst, y_tst = load_mnist(config['trn']['dataset_dir'], kind='t10k') from sklearn.model_selection import StratifiedShuffleSplit sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1) trn_idx, val_idx = list(sss.split(X, y))[0] logger.info(' There is {:d} samples in the TRN dataset.'.format( len(trn_idx))) logger.info(' There is {:d} samples in the VAL dataset.'.format( len(val_idx))) trn_ds = FashionMnistDataset(config, X[trn_idx], y[trn_idx], is_trn=True) val_ds = FashionMnistDataset(config, X[val_idx], y[val_idx], is_trn=False) if not skip_tst_dataset: tst_ds = FashionMnistDataset(config, X_tst, y_tst, is_trn=False) else: logger.warning(' Using val instead of tst!') tst_ds = val_ds epochs = int(config['trn']['epochs']) if torch.cuda.device_count() > 1: logger.warning( ' Multiplying the size of the batch by the number of GPUs (%d)!' % torch.cuda.device_count()) trn_loader = torch.utils.data.DataLoader( trn_ds, batch_size=batch_size, num_workers=12 if epochs > -1 else 1) val_loader = torch.utils.data.DataLoader( val_ds, batch_size=batch_size, shuffle=False, num_workers=12 if epochs > -1 else 1) tst_loader = torch.utils.data.DataLoader(tst_ds, batch_size=batch_size, shuffle=False, num_workers=12) return trn_loader, val_loader, tst_loader, trn_ds, val_ds, tst_ds
def main(): X_train, y_train = load_mnist('data/fashion', kind='train') X_test, y_test = load_mnist('data/fashion', kind='t10k') X_train = X_train.astype(np.uint) y_train = y_train.astype(np.uint) X_test = X_test.astype(np.uint) y_test = y_test.astype(np.uint) #q_a(X_train, y_train) #q_b(X_train, y_train) #q_b_v2(X_train, y_train) #q_c(X_train, y_train) q_d(X_train, y_train)
def dataloader(): # -------- Read data ---------# train_x, train_t = mnist_reader.load_mnist('data/', kind='train') test_x, test_t = mnist_reader.load_mnist('data/', kind='t10k') # ------ Preprocess data -----# # x_train = train_x.reshape(train_x.shape[0], 28, 28, 1).astype('float32') x_train = train_x x_train_norm = x_train / 255.0 # x_test = test_x.reshape(test_x.shape[0], 28, 28, 1).astype('float32') x_test = test_x x_test_norm = x_test / 255.0 print(np.shape(train_x), np.shape(train_t)) print(np.shape(x_train_norm), np.shape(train_t)) return x_train_norm, train_t, x_test_norm, test_t
def load_data(filePath, train=True, test=True, subsetTest=False, subsetSize=10): """ Load the FashionMNIST data set. filePath is the path to the parent directory of the fashion_mnist local repository. If train=True, returns the training set (x_train and y_train two first items in the list). If test=True, returns the full test set (x_test and y_test two last items in the list). If test=False, returns a small subset of the test set, of size subsetSize and with a balance number of images from each class (one from each with subsetSize = 10) """ ret = [] if (train == True): x_train, y_train = mnist_reader.load_mnist( filePath + 'fashion_mnist/data/fashion', kind='train') x_train = x_train.astype('float32') / 255. x_train = np.reshape( x_train, (len(x_train), 28, 28, 1)) # adapt this if using `channels_first` image data format ret.append(x_train) ret.append(y_train) x_test, y_test = mnist_reader.load_mnist(filePath + 'fashion_mnist/data/fashion', kind='t10k') x_test = x_test.astype('float32') / 255. x_test = np.reshape( x_test, (len(x_test), 28, 28, 1)) # adapt this if using `channels_first` image data format if (test == True): ret.append(x_test) ret.append(y_test) if subsetTest: x_test_small = np.zeros((subsetSize, 28, 28, 1)) y_test_small = np.zeros(subsetSize) c_len = [x_test[y_test == j].shape[0] for j in range(10)] for i in range(subsetSize): c = i % 10 x_test_small[i, :, :, :] = np.copy( x_test[y_test == c][np.random.randint(c_len[c])]) y_test_small[i] = c ret.append(x_test_small) ret.append(y_test_small) return ret
def _get_fraction_of_dataset(mnist_fraction, kind, possible_labels, epochs=1): global LABELS input_list, labels = mnist_reader.load_mnist(path=DATA_DIR, kind=kind) # take the fraction of the dataset fraction = mnist_fraction - int(np.floor(mnist_fraction)) new_size = int(round((fraction if fraction else 1) * len(input_list))) # normalize between 0 and 1 input_list = input_list[:new_size].astype(np.float32) / 255 labels = labels[:new_size] # remove labels if necessary mod_input_list = [] mod_labels = [] labels_to_remove = list(set(LABELS) - set(possible_labels)) to_remove = [True if (y in labels_to_remove) else False for y in labels] for i, (x, y) in enumerate(zip(input_list, labels)): if not to_remove[i]: mod_input_list.append(x) mod_labels.append(y) # repeat the input_list epochs times (training only) repetitions = int(np.floor(epochs)) tot_input_list = list(mod_input_list) * repetitions tot_labels = list(mod_labels) * repetitions fraction = epochs - repetitions el_to_repeat = int(round(fraction * len(mod_input_list))) input_to_repeat = mod_input_list[:el_to_repeat] labels_to_repeat = mod_labels[:el_to_repeat] tot_input_list += list(input_to_repeat) tot_labels += list(labels_to_repeat) return np.array(tot_input_list), np.array(tot_labels)
def _sanity_check(self, all_tasks): total_clf = 0 failed_clf = 0 Xt, Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k') Xt = preprocessing.StandardScaler().fit_transform(Xt.astype(float)) Xs, Ys = shuffle(Xt, Yt) num_dummy = 10 Xs = Xs[:num_dummy] Ys = [j for j in range(10)] valid_jobs = [] for v in all_tasks: clf_name = list(v.keys())[0] clf_par = list(v.values())[0] total_clf += 1 try: globals()[clf_name](**clf_par).fit(Xs, Ys) valid_jobs.append( PredictJob(clf_name, clf_par, self.num_repeat)) except Exception as e: failed_clf += 1 LOGGER.error( 'Can not create classifier "%s" with parameter "%s". Reason: %s' % (clf_name, clf_par, e)) LOGGER.info('%d classifiers to test, %d fail to create!' % (total_clf, failed_clf)) return valid_jobs
def test(self, test_data_path): with open("train.npy", "rb") as file: self.w = np.load(file,allow_pickle=True) with open("train_label.npy", "rb") as file: self.w_index = np.load(file,allow_pickle=True) self.images, images_labels = mnist_reader.load_mnist(self.data_path, kind='train') images, labels = mnist_reader.load_mnist(test_data_path, kind='t10k') error = 0 print('Testing...........') for i in range(len(images)): print(i) if i==1000:break guessed_label = self.perceptron_belong(images[i]) if guessed_label != labels[i]: error += 1 print('finished testing . . .') return 100 - error / len(images) * 100
def test(self, test_data_path): images, labels = mnist_reader.load_mnist(test_data_path, kind='t10k') error = 0 for i in range(len(images)): guessed_label = self.perceptron_belong(np.reshape(images[i], (784,))) if guessed_label != labels[i]: error += 1 return 100 - error / len(images) * 100
def load(self, folder_path, data_type): """ This function loads the data-set :param folder_path: path to data-set folder :param data_type: train or test data :return: data and labels """ train_data, test_data = mnist_reader.load_mnist(folder_path, kind=data_type) return train_data, test_data
def load_images(path, test_percent, valid_percent, mnist=True): datasets = {} if mnist: x_train, y_train = mnist_reader.load_mnist(path, kind='train') x_test, y_test = mnist_reader.load_mnist(path, kind='t10k') train_labels = y_train.reshape((y_train.shape[0], 1)) test_labels = y_test.reshape((y_test.shape[0], 1)) train_inputs = x_train / 255 test_inputs = x_test / 255 datasets = { 'train_inputs': train_inputs, 'train_labels': train_labels, 'test_inputs': test_inputs, 'test_labels': test_labels } return datasets
def main(): x_train, y_train = load_mnist('data/fashion', kind='train') x_test, y_test = load_mnist('data/fashion', kind='t10k') x_train, y_train = shuffle(x_train, y_train) x_test, y_test = shuffle(x_test, y_test) if LESS_DATA: x_train, y_train = x_train[0:10000, ], y_train[0:10000, ] x_test, y_test = x_test[0:1000, ], y_test[0:1000, ] num_classes = len(set(y_train)) n = x_train.shape[0] n_batches = n // BATCH_SIZE graph = tf.Graph() with graph.as_default(): graph_def = tf.GraphDef() with gfile.FastGFile(PB_FILE, 'rb') as f: graph_def.ParseFromString(f.read()) tf.import_graph_def(graph_def, name="pb") # tf.summary.FileWriter("tensor_board/", graph=graph) graph_nodes = [n.name for n in graph.as_graph_def().node] print(graph_nodes) input_tensor = graph.get_tensor_by_name('pb/input:0') output_tensor = graph.get_tensor_by_name('pb/output_dense/BiasAdd:0') with tf.Session(graph=graph) as sess: output_result = sess.run(output_tensor, feed_dict={input_tensor: x_test}) output_logit = np.argmax(output_result, axis=1) print("pb file result: {}".format(accuracy_score(y_test, output_logit))) # 任務:抽換dense layer 從128 個node 換成64 個node target_tensor = graph.get_tensor_by_name('pb/my_layer/dense/bias:0') with tf.Session(graph=graph) as sess: sess.run(tf.assign(target_tensor, np.zeros(128)))
def load_data(self): train_data_raw, self.train_labels = mnist_reader.load_mnist( 'data/fashion', kind='train') test_data_raw, self.test_labels = mnist_reader.load_mnist( 'data/fashion', kind='t10k') train_data_raw = train_data_raw.reshape( (train_data_raw.shape[0], int(np.sqrt(train_data_raw.shape[1])), int(np.sqrt(train_data_raw.shape[1])))) test_data_raw = test_data_raw.reshape( (test_data_raw.shape[0], int(np.sqrt(test_data_raw.shape[1])), int(np.sqrt(test_data_raw.shape[1])))) train_data_raw = train_data_raw[:, :, :, np.newaxis] test_data_raw = test_data_raw[:, :, :, np.newaxis] self.train_data = train_data_raw / 255.0 self.test_data = test_data_raw / 255.0 self.train_labels = np_utils.to_categorical(self.train_labels, 10) self.test_labels = np_utils.to_categorical(self.test_labels, 10)
def train(self): images, labels = mnist_reader.load_mnist(self.data_path, kind='train') self.images = np.array([]) for img in images: a = abs(np.fft.fft2(np.reshape(img, (28, 28)))) np.append(self.images, a); for e in range(self.epoch): for i in range(len(images)): print(i) self.perceptron_check(np.reshape(images[i], (784,)), labels[i]) print(self.w)
def train(self): print('learning...........') self.images, labels = mnist_reader.load_mnist(self.data_path, kind='train') for e in range(self.epoch): for i in range(len(self.images)): print(i) # if i == 2000: break self.perceptron_check(self.images[i],i, labels[i]) print(self.w) with open("train.npy","wb") as file: np.save(file, self.w,allow_pickle=True) with open("train_label.npy" ,"wb") as file: np.save(file, self.w_index,allow_pickle=True)
def train_test_data_loading(self): ############################### # import fashion_mnist data # ############################### x_train, y_train = mnist_reader.load_mnist('/app/data/fashion', kind='train') x_test, y_test = mnist_reader.load_mnist('/app/data/fashion', kind='t10k') self.prt_obj.data_dimension_descripition(x_train, x_test) self.prt_obj.data_dimension_descripition(y_train, y_test) # otherwise, can import tensorflow build-in dataset by: # from tensorflow.examples.tutorials.mnist import input_data # data = input_data.read_data_set('data/fashion') # or data = input_data.read_data_sets('data/fashion', source_url='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/') # data.train.next_batch(BATCH_SIZE) # Keras build-in dataset by: # fashion_mnist = keras.dataset.fashion_mnist # (train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data() return (x_train, y_train), (x_test, y_test)
def main(): #Load data from dataset X_train, y_train = load_mnist( DATADIR, 'train') #X_train=60000 images, each 28x28; y_train = 60000 labels X_test, y_test = load_mnist( DATADIR, 't10k') #X_train=10000 images, each 28x28; y_train = 10000 labels X_train = X_train.reshape(X_train.shape[0], IPIMG_H, IPIMG_W, 1) #y_train = y_train.reshape(y_train.shape[0],IPIMG_H,IPIMG_W) X_test = X_test.reshape(X_test.shape[0], IPIMG_H, IPIMG_W, 1) #y_test = y_test.reshape(y_test.shape[0],IPIMG_H,IPIMG_W) #Construct a model model = Sequential() model.add( Conv2D(32, kernel_size=(3, 3), data_format='channels_last', input_shape=(IPIMG_H, IPIMG_W, 1), activation='relu')) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dense(LBLCNT, activation='softmax')) model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit(X_train, y_train, epochs=5, validation_data=(X_test, y_test)) #steps_per_epoch=10, #validation_steps=100) score = model.evaluate(X_test, y_test, steps=50) print('Test score:', score[0]) print('Test accuracy:', score[1])
def hamming_tests(): X_train, y_train = mnist_reader.load_mnist('../data/fashion', kind='train') X_test, y_test = mnist_reader.load_mnist('../data/fashion', kind='t10k') X_train, X_test = X_train.astype('float32') / 255, X_test.astype( 'float32') / 255 best_threshold = 0.1 the_best_error = 100 for i in range(1, 10): threshold = i / 10 X_train_bin, X_test_bin = binarize(X_train, threshold), binarize( X_test, threshold) k_values = [3, 4, 5, 6, 7, 8, 9] error_best, best_k, errors = model_selection_knn_hamming( X_test_bin, X_train_bin, y_test, y_train, k_values) print("threshold", threshold, " acc", 1 - error_best, " best_k: ", best_k) if error_best < the_best_error: the_best_error = error_best best_threshold = threshold print("best acc: ", 1 - the_best_error) print("best threshold: ", best_threshold)
def main(): X_test, y_test = mnist_reader.load_mnist('data/mnist', kind='t10k') X_test, y_test = classSeparator(1, 8, X_test, y_test) k = [ 0, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255 ] ''' for i in range(len(k)): X_train, y_train = mnist_reader.load_mnist('data/mnist', kind='train') X_test, y_test = mnist_reader.load_mnist('data/mnist', kind='t10k') X_train = binarization(k[i],X_train) X_test = binarization(k[i],X_test) X_train, y_train = classSeparator(1,8,X_train,y_train) X_test, y_test = classSeparator(1,8,X_test,y_test) print ("Class Separated!") summaries = summarizeByClass(X_train,y_train) print ("Summaries Done!") pickleUnload("Scrap/nb_"+str(k[i])+"_model.pkl",summaries) summaries = pickleLoad("Scrap/nb_"+str(k[i])+"_model.pkl") predictions = getPredictions(summaries, X_test) print ("Predictions Done!") pickleUnload("Scrap/nb_"+str(k[i])+"_predictions.pkl",predictions) ''' tpr = [] fpr = [] predictions = pickleLoad("Scrap/nb_" + str(3) + "_predictions.pkl") fpr, tpr, thresholds = roc_curve(y_test, predictions, pos_label='1') roc_auc = auc(y_test, predictions) # Plot ROC curve plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % roc_auc) plt.plot([0, 1], [0, 1], 'k--') # random predictions curve plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.0]) plt.xlabel('False Positive Rate or (1 - Specifity)') plt.ylabel('True Positive Rate or (Sensitivity)') plt.title('Receiver Operating Characteristic') plt.legend(loc="lower right") plt.show() '''
def _sanity_check(self, all_tasks): total_clf = 0 failed_clf = 0 Xt, Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k') Xt = preprocessing.StandardScaler().fit_transform(Xt) Xs, Ys = shuffle(Xt, Yt) num_dummy = 10 Xs = Xs[:num_dummy] Ys = [j for j in range(10)] valid_jobs = [] for v in all_tasks: clf_name = list(v.keys())[0] clf_par = list(v.values())[0] total_clf += 1 try: globals()[clf_name](**clf_par).fit(Xs, Ys) valid_jobs.append(PredictJob(clf_name, clf_par, self.num_repeat)) except Exception as e: failed_clf += 1 LOGGER.error('Can not create classifier "%s" with parameter "%s". Reason: %s' % (clf_name, clf_par, e)) LOGGER.info('%d classifiers to test, %d fail to create!' % (total_clf, failed_clf)) return valid_jobs
predicted_label = self.predict(input) if predicted_label == label: return 1 self.w[predicted_label] = np.subtract(self.w[predicted_label], input) self.biases[predicted_label] -= 1 self.w[label] = np.add(self.w[label], input) self.biases[label] += 1 return 0 def predict(self, input): index = np.argmax([(np.dot(self.w[i], input) + self.biases[i]) for i in range(len(self.labels))]) return self.labels[index] X_train, y_train = mnist_reader.load_mnist('../data/fashion', kind='train') X_test, y_test = mnist_reader.load_mnist('../data/fashion', kind='t10k') features = 28 * 28 + 5 + 28 + 28 perceptron = Perceptron([i for i in range(10)], features) epochs = 30 for j in range(epochs): correct = 0 for i in range(len(X_train)): f = feature_extract(X_train[i], i, train=True) correct += perceptron.train(f, y_train[i]) print(j, correct / len(X_train)) x = 0 for i in range(len(X_test)):
import matplotlib.pyplot as plt import numpy as np import tensorflow as tf from tensorflow.contrib.tensorboard.plugins import projector from configs import DATA_DIR, LOG_DIR, VIS_DIR from utils import mnist_reader from utils.helper import get_sprite_image X, Y = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k') labels = ['t_shirt_top', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt', 'sneaker', 'bag', 'ankle_boots'] Y_str = np.array([labels[j] for j in Y]) np.savetxt('Xtest.tsv', X, fmt='%.6e', delimiter='\t') np.savetxt('Ytest.tsv', Y_str, fmt='%s') plt.imsave('zalando-mnist-sprite.png', get_sprite_image(X), cmap='gray') exit() embedding_var = tf.Variable(X, name='mnist_pixels') # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = VIS_DIR + 'Ytest.tsv' embedding.sprite.image_path = VIS_DIR + 'zalando-mnist-sprite.png' # Specify the width and height of a single thumbnail.