def main():
args = parse_option()
print(args)
x_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train')
x_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k')
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
x_train=x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test=x_test.reshape(x_test.shape[0], 28, 28 ,1)
x_train = x_train / 255.0
x_test = x_test / 255.0
y_train=keras.utils.to_categorical(y_train)
y_test=keras.utils.to_categorical(y_test)
num_classes = 10
print("Grid search for batch_size,batch norm and learning rate")
model = KerasClassifier(build_fn=build_model,,epochs=40,verbose=1)
batch_size = [32,64,128]
lr = [0.01,0.001]
use_bn = [True,False]
param_grid = dict(batch_size=batch_size, lr=lr,use_bn=use_bn)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
grid_result = grid.fit(x_train, y_train)
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
def load_data():
print("Loading data..")
X_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k')
# we want to work on a binary classification problem so only class 5 and class 7 will be used
tr_mask = np.where(np.logical_or(y_train == 5, y_train == 7))[0]
te_mask = np.where(np.logical_or(y_test == 5, y_test == 7))[0]
X_train = X_train[tr_mask, :]
y_train = y_train[tr_mask]
X_test = X_test[te_mask, :]
y_test = y_test[te_mask]
# let class 5 be 0 and let class 7 be 1
y_train[y_train == 5] = 0
y_train[y_train == 7] = 1
y_test[y_test == 5] = 0
y_test[y_test == 7] = 1
# Debugging
assert X_train.shape[0] == y_train.shape[0]
assert X_test.shape[0] == y_test.shape[0]
# plt.imshow(X_train[1251].reshape(28,28),cmap=plt.cm.gray)
# plt.show()
print("Done..")
return X_train, y_train, X_test, y_test
def __init__(self, part='train', batch_size=128, partial=1000):
from utils.mnist_reader import load_mnist
train_raw, _ = load_mnist('datasets/fashion', kind='train')
test_raw, _ = load_mnist('datasets/fashion', kind='t10k')
grey_scale = lambda x: np.float32(x / 255.)
def static_binarize(d):
ids = d < 0.5
d[ids] = 0.
d[~ids] = 1.
train_data = grey_scale(train_raw)
test_data = grey_scale(test_raw)
static_binarize(train_data)
static_binarize(test_data)
assert train_data.shape == (60000, 784)
assert test_data.shape == (10000, 784)
self.data = {
'train': train_data[:55000],
'valid': train_data[55000:],
'test': test_data,
'partial_train': train_data[:partial],
'partial_test': test_data[:partial],
}[part]
self.size = self.data.shape[0]
self.batch_size = batch_size
self._construct()
def __init__(self, pending_q: Queue) -> None:
super().__init__()
self.pending_q = pending_q
X, self.Y = mnist_reader.load_mnist(path=DATA_DIR, kind='train')
Xt, self.Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k')
scaler = preprocessing.StandardScaler().fit(X)
self.X = scaler.transform(X)
self.Xt = scaler.transform(Xt)
def __init__(self, pending_q: Queue) -> None:
super().__init__()
self.pending_q = pending_q
X, self.Y = mnist_reader.load_mnist(path=DATA_DIR, kind='train')
Xt, self.Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k')
scaler = preprocessing.StandardScaler().fit(X.astype(float))
self.X = scaler.transform(X.astype(float))
self.Xt = scaler.transform(Xt.astype(float))
def init():
x_train, y_train = mnist_reader.load_mnist('data/fashion', kind='train')
x_test, y_test = mnist_reader.load_mnist('data/fashion', kind='t10k')
x_train = np.array(x_train)
y_train = np.array(y_train)
x_test = np.array(x_test)
y_test = np.array(y_test)
x_train_temp = []
y_train_temp = []
x_test_temp = []
y_test_temp = []
sandal_count = 0
sneaker_count = 0
train_len = len(x_train)
test_len = len(x_test)
# Convert the training data into a binary classification problem.
for i in range(train_len):
# Let sandal data be class 0.
if y_train[i] == 5 and sandal_count < 3000:
y_train_temp.append(0)
x_train_temp.append(x_train[i])
sandal_count = sandal_count + 1
# Let sneaker data be class 1.
if y_train[i] == 7 and sneaker_count < 3000:
y_train_temp.append(1)
x_train_temp.append(x_train[i])
sneaker_count = sneaker_count + 1
# Convert the test data into a binary classification problem.
for i in range(test_len):
# Let sandal data be class 0.
if y_test[i] == 5:
y_test_temp.append(0)
x_test_temp.append(x_test[i])
# Let sneaker data be class 1.
if y_test[i] == 7:
y_test_temp.append(1)
x_test_temp.append(x_test[i])
x_train = np.array(x_train_temp) / 255
y_train = y_train_temp
x_test = np.array(x_test_temp) / 255
y_test = y_test_temp
print("Training set size: " + str(len(x_train)))
print("Test set size: " + str(len(x_test)))
return x_train, y_train, x_test, y_test
def main():
x_train, y_train = load_mnist('data/fashion', kind='train')
x_test, y_test = load_mnist('data/fashion', kind='t10k')
x_train, y_train = shuffle(x_train, y_train)
x_test, y_test = shuffle(x_test, y_test)
if LESS_DATA:
x_train, y_train = x_train[0:10000, ], y_train[0:10000, ]
x_test, y_test = x_test[0:1000, ], y_train[0:1000, ]
num_classes = len(set(y_train))
# One-hot encoding
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
input_img = Input(shape=(28, 28, 1))
x = Conv2D(32, kernel_size=(3, 3), activation='relu')(input_img)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Dropout(0.25)(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
output = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=input_img, outputs=output)
model.summary()
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
# convert keras model to pb files
output_names = [out.op.name for out in model.outputs]
frozen_graph = utils.freeze_session(keras.backend.get_session(),
output_names=output_names)
tf.train.write_graph(frozen_graph,
"output_models/",
"keras_fashion.pb",
as_text=False)
tf.train.write_graph(frozen_graph,
"output_models/",
"keras_fashion.pbtxt",
as_text=True)
def _load_data(self):
X_train, y_train = load_mnist('./../../submodules/fashion-mnist/data/fashion', kind='train')
X_test, y_test = load_mnist('./../../submodules/fashion-mnist/data/fashion/', kind='t10k')
train_data = X_train
train_target = y_train
test_data = X_test
test_target = y_test
return train_data, train_target, test_data, test_target
def load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
X_train, y_train = load_mnist('./../../submodules/fashion-mnist/data/fashion', kind='train')
X_test, y_test = load_mnist('./../../submodules/fashion-mnist/data/fashion/', kind='t10k')
train_data = self._normalise(X_train)
train_target = y_train
test_data = self._normalise(X_test)
test_target = y_test
return train_data, train_target, test_data, test_target
def euclidean_tests():
X_train, y_train = mnist_reader.load_mnist('../data/fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('../data/fashion', kind='t10k')
X_train, X_test = X_train.astype('float32') / 255, X_test.astype(
'float32') / 255
k_values = [3, 4, 5, 6, 7, 8, 9]
print("StartTime =", datetime.now().strftime("%H:%M:%S"))
error_best, best_k, errors = model_selection_knn_euclidean(
X_test, X_train, y_test, y_train, k_values)
print("EndTime =", datetime.now().strftime("%H:%M:%S"))
print("error: ", error_best, "best_k: ", best_k)
def import_data():
X_train_mnist, y_train_mnist = mnist_reader.load_mnist('data/fashion',
kind='train')
X_test_mnist, y_test_mnist = mnist_reader.load_mnist('data/fashion',
kind='t10k')
for i in range(y_train_mnist.size):
if y_train_mnist[i] < 5:
y_train_.append(y_train_mnist[i])
x_train_.append(X_train_mnist[i])
return np.array(x_train_), np.array(y_train_)
def readfile(dir):
train_img, train_label = mnist_reader.load_mnist(path=dir, kind='train')
test_img, test_label = mnist_reader.load_mnist(path=dir, kind='t10k')
mu=np.mean(train_img, axis=0, dtype=np.float64)
x=train_img-mu
cov=np.dot(np.transpose(x), x)
cov=np.divide(cov, len(x))
[egval, egvec]=la.eig(cov)
idx=np.argsort(egval)
egval=egval[idx]
egvec=egvec[:, idx]
return [egvec, train_img, test_img, train_label, test_label]
def prepare_loaders(config,
logger,
skip_tst_dataset,
multiply_by_number_of_gpu=False):
from radim.FashionMnistDataset import FashionMnistDataset
batch_size = int(config['trn']['batch_size'])
if multiply_by_number_of_gpu:
batch_size = torch.cuda.device_count() * int(
config['trn']['batch_size'])
logger.info(" Batch size: %d" % batch_size)
from utils.mnist_reader import load_mnist
X, y = load_mnist(config['trn']['dataset_dir'], kind='train')
X_tst, y_tst = load_mnist(config['trn']['dataset_dir'], kind='t10k')
from sklearn.model_selection import StratifiedShuffleSplit
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1)
trn_idx, val_idx = list(sss.split(X, y))[0]
logger.info(' There is {:d} samples in the TRN dataset.'.format(
len(trn_idx)))
logger.info(' There is {:d} samples in the VAL dataset.'.format(
len(val_idx)))
trn_ds = FashionMnistDataset(config, X[trn_idx], y[trn_idx], is_trn=True)
val_ds = FashionMnistDataset(config, X[val_idx], y[val_idx], is_trn=False)
if not skip_tst_dataset:
tst_ds = FashionMnistDataset(config, X_tst, y_tst, is_trn=False)
else:
logger.warning(' Using val instead of tst!')
tst_ds = val_ds
epochs = int(config['trn']['epochs'])
if torch.cuda.device_count() > 1:
logger.warning(
' Multiplying the size of the batch by the number of GPUs (%d)!' %
torch.cuda.device_count())
trn_loader = torch.utils.data.DataLoader(
trn_ds, batch_size=batch_size, num_workers=12 if epochs > -1 else 1)
val_loader = torch.utils.data.DataLoader(
val_ds,
batch_size=batch_size,
shuffle=False,
num_workers=12 if epochs > -1 else 1)
tst_loader = torch.utils.data.DataLoader(tst_ds,
batch_size=batch_size,
shuffle=False,
num_workers=12)
return trn_loader, val_loader, tst_loader, trn_ds, val_ds, tst_ds
def main():
X_train, y_train = load_mnist('data/fashion', kind='train')
X_test, y_test = load_mnist('data/fashion', kind='t10k')
X_train = X_train.astype(np.uint)
y_train = y_train.astype(np.uint)
X_test = X_test.astype(np.uint)
y_test = y_test.astype(np.uint)
#q_a(X_train, y_train)
#q_b(X_train, y_train)
#q_b_v2(X_train, y_train)
#q_c(X_train, y_train)
q_d(X_train, y_train)
def dataloader():
# -------- Read data ---------#
train_x, train_t = mnist_reader.load_mnist('data/', kind='train')
test_x, test_t = mnist_reader.load_mnist('data/', kind='t10k')
# ------ Preprocess data -----#
# x_train = train_x.reshape(train_x.shape[0], 28, 28, 1).astype('float32')
x_train = train_x
x_train_norm = x_train / 255.0
# x_test = test_x.reshape(test_x.shape[0], 28, 28, 1).astype('float32')
x_test = test_x
x_test_norm = x_test / 255.0
print(np.shape(train_x), np.shape(train_t))
print(np.shape(x_train_norm), np.shape(train_t))
return x_train_norm, train_t, x_test_norm, test_t
def load_data(filePath,
train=True,
test=True,
subsetTest=False,
subsetSize=10):
"""
Load the FashionMNIST data set.
filePath is the path to the parent directory of the fashion_mnist local repository.
If train=True, returns the training set (x_train and y_train two first items in the list).
If test=True, returns the full test set (x_test and y_test two last items in the list).
If test=False, returns a small subset of the test set, of size subsetSize and with a balance number of images from each class (one from each with subsetSize = 10)
"""
ret = []
if (train == True):
x_train, y_train = mnist_reader.load_mnist(
filePath + 'fashion_mnist/data/fashion', kind='train')
x_train = x_train.astype('float32') / 255.
x_train = np.reshape(
x_train,
(len(x_train), 28, 28,
1)) # adapt this if using `channels_first` image data format
ret.append(x_train)
ret.append(y_train)
x_test, y_test = mnist_reader.load_mnist(filePath +
'fashion_mnist/data/fashion',
kind='t10k')
x_test = x_test.astype('float32') / 255.
x_test = np.reshape(
x_test, (len(x_test), 28, 28,
1)) # adapt this if using `channels_first` image data format
if (test == True):
ret.append(x_test)
ret.append(y_test)
if subsetTest:
x_test_small = np.zeros((subsetSize, 28, 28, 1))
y_test_small = np.zeros(subsetSize)
c_len = [x_test[y_test == j].shape[0] for j in range(10)]
for i in range(subsetSize):
c = i % 10
x_test_small[i, :, :, :] = np.copy(
x_test[y_test == c][np.random.randint(c_len[c])])
y_test_small[i] = c
ret.append(x_test_small)
ret.append(y_test_small)
return ret
def _get_fraction_of_dataset(mnist_fraction, kind, possible_labels, epochs=1):
global LABELS
input_list, labels = mnist_reader.load_mnist(path=DATA_DIR, kind=kind)
# take the fraction of the dataset
fraction = mnist_fraction - int(np.floor(mnist_fraction))
new_size = int(round((fraction if fraction else 1) * len(input_list)))
# normalize between 0 and 1
input_list = input_list[:new_size].astype(np.float32) / 255
labels = labels[:new_size]
# remove labels if necessary
mod_input_list = []
mod_labels = []
labels_to_remove = list(set(LABELS) - set(possible_labels))
to_remove = [True if (y in labels_to_remove) else False for y in labels]
for i, (x, y) in enumerate(zip(input_list, labels)):
if not to_remove[i]:
mod_input_list.append(x)
mod_labels.append(y)
# repeat the input_list epochs times (training only)
repetitions = int(np.floor(epochs))
tot_input_list = list(mod_input_list) * repetitions
tot_labels = list(mod_labels) * repetitions
fraction = epochs - repetitions
el_to_repeat = int(round(fraction * len(mod_input_list)))
input_to_repeat = mod_input_list[:el_to_repeat]
labels_to_repeat = mod_labels[:el_to_repeat]
tot_input_list += list(input_to_repeat)
tot_labels += list(labels_to_repeat)
return np.array(tot_input_list), np.array(tot_labels)
def _sanity_check(self, all_tasks):
total_clf = 0
failed_clf = 0
Xt, Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k')
Xt = preprocessing.StandardScaler().fit_transform(Xt.astype(float))
Xs, Ys = shuffle(Xt, Yt)
num_dummy = 10
Xs = Xs[:num_dummy]
Ys = [j for j in range(10)]
valid_jobs = []
for v in all_tasks:
clf_name = list(v.keys())[0]
clf_par = list(v.values())[0]
total_clf += 1
try:
globals()[clf_name](**clf_par).fit(Xs, Ys)
valid_jobs.append(
PredictJob(clf_name, clf_par, self.num_repeat))
except Exception as e:
failed_clf += 1
LOGGER.error(
'Can not create classifier "%s" with parameter "%s". Reason: %s'
% (clf_name, clf_par, e))
LOGGER.info('%d classifiers to test, %d fail to create!' %
(total_clf, failed_clf))
return valid_jobs
def test(self, test_data_path):
with open("train.npy", "rb") as file:
self.w = np.load(file,allow_pickle=True)
with open("train_label.npy", "rb") as file:
self.w_index = np.load(file,allow_pickle=True)
self.images, images_labels = mnist_reader.load_mnist(self.data_path, kind='train')
images, labels = mnist_reader.load_mnist(test_data_path, kind='t10k')
error = 0
print('Testing...........')
for i in range(len(images)):
print(i)
if i==1000:break
guessed_label = self.perceptron_belong(images[i])
if guessed_label != labels[i]: error += 1
print('finished testing . . .')
return 100 - error / len(images) * 100
def test(self, test_data_path):
images, labels = mnist_reader.load_mnist(test_data_path, kind='t10k')
error = 0
for i in range(len(images)):
guessed_label = self.perceptron_belong(np.reshape(images[i], (784,)))
if guessed_label != labels[i]: error += 1
return 100 - error / len(images) * 100
def load(self, folder_path, data_type):
"""
This function loads the data-set
:param folder_path: path to data-set folder
:param data_type: train or test data
:return: data and labels
"""
train_data, test_data = mnist_reader.load_mnist(folder_path, kind=data_type)
return train_data, test_data
def load_images(path, test_percent, valid_percent, mnist=True):
datasets = {}
if mnist:
x_train, y_train = mnist_reader.load_mnist(path, kind='train')
x_test, y_test = mnist_reader.load_mnist(path, kind='t10k')
train_labels = y_train.reshape((y_train.shape[0], 1))
test_labels = y_test.reshape((y_test.shape[0], 1))
train_inputs = x_train / 255
test_inputs = x_test / 255
datasets = {
'train_inputs': train_inputs,
'train_labels': train_labels,
'test_inputs': test_inputs,
'test_labels': test_labels
}
return datasets
def main():
x_train, y_train = load_mnist('data/fashion', kind='train')
x_test, y_test = load_mnist('data/fashion', kind='t10k')
x_train, y_train = shuffle(x_train, y_train)
x_test, y_test = shuffle(x_test, y_test)
if LESS_DATA:
x_train, y_train = x_train[0:10000, ], y_train[0:10000, ]
x_test, y_test = x_test[0:1000, ], y_test[0:1000, ]
num_classes = len(set(y_train))
n = x_train.shape[0]
n_batches = n // BATCH_SIZE
graph = tf.Graph()
with graph.as_default():
graph_def = tf.GraphDef()
with gfile.FastGFile(PB_FILE, 'rb') as f:
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def, name="pb")
# tf.summary.FileWriter("tensor_board/", graph=graph)
graph_nodes = [n.name for n in graph.as_graph_def().node]
print(graph_nodes)
input_tensor = graph.get_tensor_by_name('pb/input:0')
output_tensor = graph.get_tensor_by_name('pb/output_dense/BiasAdd:0')
with tf.Session(graph=graph) as sess:
output_result = sess.run(output_tensor,
feed_dict={input_tensor: x_test})
output_logit = np.argmax(output_result, axis=1)
print("pb file result: {}".format(accuracy_score(y_test,
output_logit)))
# 任務：抽換dense layer 從128 個node 換成64 個node
target_tensor = graph.get_tensor_by_name('pb/my_layer/dense/bias:0')
with tf.Session(graph=graph) as sess:
sess.run(tf.assign(target_tensor, np.zeros(128)))
def load_data(self):
train_data_raw, self.train_labels = mnist_reader.load_mnist(
'data/fashion', kind='train')
test_data_raw, self.test_labels = mnist_reader.load_mnist(
'data/fashion', kind='t10k')
train_data_raw = train_data_raw.reshape(
(train_data_raw.shape[0], int(np.sqrt(train_data_raw.shape[1])),
int(np.sqrt(train_data_raw.shape[1]))))
test_data_raw = test_data_raw.reshape(
(test_data_raw.shape[0], int(np.sqrt(test_data_raw.shape[1])),
int(np.sqrt(test_data_raw.shape[1]))))
train_data_raw = train_data_raw[:, :, :, np.newaxis]
test_data_raw = test_data_raw[:, :, :, np.newaxis]
self.train_data = train_data_raw / 255.0
self.test_data = test_data_raw / 255.0
self.train_labels = np_utils.to_categorical(self.train_labels, 10)
self.test_labels = np_utils.to_categorical(self.test_labels, 10)
def train(self):
images, labels = mnist_reader.load_mnist(self.data_path, kind='train')
self.images = np.array([])
for img in images:
a = abs(np.fft.fft2(np.reshape(img, (28, 28))))
np.append(self.images, a);
for e in range(self.epoch):
for i in range(len(images)):
print(i)
self.perceptron_check(np.reshape(images[i], (784,)), labels[i])
print(self.w)
def train(self):
print('learning...........')
self.images, labels = mnist_reader.load_mnist(self.data_path, kind='train')
for e in range(self.epoch):
for i in range(len(self.images)):
print(i)
# if i == 2000: break
self.perceptron_check(self.images[i],i, labels[i])
print(self.w)
with open("train.npy","wb") as file:
np.save(file, self.w,allow_pickle=True)
with open("train_label.npy" ,"wb") as file:
np.save(file, self.w_index,allow_pickle=True)
def train_test_data_loading(self):
###############################
# import fashion_mnist data #
###############################
x_train, y_train = mnist_reader.load_mnist('/app/data/fashion', kind='train')
x_test, y_test = mnist_reader.load_mnist('/app/data/fashion', kind='t10k')
self.prt_obj.data_dimension_descripition(x_train, x_test)
self.prt_obj.data_dimension_descripition(y_train, y_test)
# otherwise, can import tensorflow build-in dataset by:
# from tensorflow.examples.tutorials.mnist import input_data
# data = input_data.read_data_set('data/fashion')
# or data = input_data.read_data_sets('data/fashion', source_url='http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/')
# data.train.next_batch(BATCH_SIZE)
# Keras build-in dataset by:
# fashion_mnist = keras.dataset.fashion_mnist
# (train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
return (x_train, y_train), (x_test, y_test)
def main():
#Load data from dataset
X_train, y_train = load_mnist(
DATADIR,
'train') #X_train=60000 images, each 28x28; y_train = 60000 labels
X_test, y_test = load_mnist(
DATADIR,
't10k') #X_train=10000 images, each 28x28; y_train = 10000 labels
X_train = X_train.reshape(X_train.shape[0], IPIMG_H, IPIMG_W, 1)
#y_train = y_train.reshape(y_train.shape[0],IPIMG_H,IPIMG_W)
X_test = X_test.reshape(X_test.shape[0], IPIMG_H, IPIMG_W, 1)
#y_test = y_test.reshape(y_test.shape[0],IPIMG_H,IPIMG_W)
#Construct a model
model = Sequential()
model.add(
Conv2D(32,
kernel_size=(3, 3),
data_format='channels_last',
input_shape=(IPIMG_H, IPIMG_W, 1),
activation='relu'))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(LBLCNT, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X_train, y_train, epochs=5, validation_data=(X_test, y_test))
#steps_per_epoch=10,
#validation_steps=100)
score = model.evaluate(X_test, y_test, steps=50)
print('Test score:', score[0])
print('Test accuracy:', score[1])
def hamming_tests():
X_train, y_train = mnist_reader.load_mnist('../data/fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('../data/fashion', kind='t10k')
X_train, X_test = X_train.astype('float32') / 255, X_test.astype(
'float32') / 255
best_threshold = 0.1
the_best_error = 100
for i in range(1, 10):
threshold = i / 10
X_train_bin, X_test_bin = binarize(X_train, threshold), binarize(
X_test, threshold)
k_values = [3, 4, 5, 6, 7, 8, 9]
error_best, best_k, errors = model_selection_knn_hamming(
X_test_bin, X_train_bin, y_test, y_train, k_values)
print("threshold", threshold, " acc", 1 - error_best, " best_k: ",
best_k)
if error_best < the_best_error:
the_best_error = error_best
best_threshold = threshold
print("best acc: ", 1 - the_best_error)
print("best threshold: ", best_threshold)
def main():
X_test, y_test = mnist_reader.load_mnist('data/mnist', kind='t10k')
X_test, y_test = classSeparator(1, 8, X_test, y_test)
k = [
0, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225,
240, 255
]
'''
for i in range(len(k)):
X_train, y_train = mnist_reader.load_mnist('data/mnist', kind='train')
X_test, y_test = mnist_reader.load_mnist('data/mnist', kind='t10k')
X_train = binarization(k[i],X_train)
X_test = binarization(k[i],X_test)
X_train, y_train = classSeparator(1,8,X_train,y_train)
X_test, y_test = classSeparator(1,8,X_test,y_test)
print ("Class Separated!")
summaries = summarizeByClass(X_train,y_train)
print ("Summaries Done!")
pickleUnload("Scrap/nb_"+str(k[i])+"_model.pkl",summaries)
summaries = pickleLoad("Scrap/nb_"+str(k[i])+"_model.pkl")
predictions = getPredictions(summaries, X_test)
print ("Predictions Done!")
pickleUnload("Scrap/nb_"+str(k[i])+"_predictions.pkl",predictions)
'''
tpr = []
fpr = []
predictions = pickleLoad("Scrap/nb_" + str(3) + "_predictions.pkl")
fpr, tpr, thresholds = roc_curve(y_test, predictions, pos_label='1')
roc_auc = auc(y_test, predictions)
# Plot ROC curve
plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % roc_auc)
plt.plot([0, 1], [0, 1], 'k--') # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate or (1 - Specifity)')
plt.ylabel('True Positive Rate or (Sensitivity)')
plt.title('Receiver Operating Characteristic')
plt.legend(loc="lower right")
plt.show()
'''
def _sanity_check(self, all_tasks):
total_clf = 0
failed_clf = 0
Xt, Yt = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k')
Xt = preprocessing.StandardScaler().fit_transform(Xt)
Xs, Ys = shuffle(Xt, Yt)
num_dummy = 10
Xs = Xs[:num_dummy]
Ys = [j for j in range(10)]
valid_jobs = []
for v in all_tasks:
clf_name = list(v.keys())[0]
clf_par = list(v.values())[0]
total_clf += 1
try:
globals()[clf_name](**clf_par).fit(Xs, Ys)
valid_jobs.append(PredictJob(clf_name, clf_par, self.num_repeat))
except Exception as e:
failed_clf += 1
LOGGER.error('Can not create classifier "%s" with parameter "%s". Reason: %s' % (clf_name, clf_par, e))
LOGGER.info('%d classifiers to test, %d fail to create!' % (total_clf, failed_clf))
return valid_jobs
predicted_label = self.predict(input)
if predicted_label == label:
return 1
self.w[predicted_label] = np.subtract(self.w[predicted_label], input)
self.biases[predicted_label] -= 1
self.w[label] = np.add(self.w[label], input)
self.biases[label] += 1
return 0
def predict(self, input):
index = np.argmax([(np.dot(self.w[i], input) + self.biases[i])
for i in range(len(self.labels))])
return self.labels[index]
X_train, y_train = mnist_reader.load_mnist('../data/fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('../data/fashion', kind='t10k')
features = 28 * 28 + 5 + 28 + 28
perceptron = Perceptron([i for i in range(10)], features)
epochs = 30
for j in range(epochs):
correct = 0
for i in range(len(X_train)):
f = feature_extract(X_train[i], i, train=True)
correct += perceptron.train(f, y_train[i])
print(j, correct / len(X_train))
x = 0
for i in range(len(X_test)):
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector
from configs import DATA_DIR, LOG_DIR, VIS_DIR
from utils import mnist_reader
from utils.helper import get_sprite_image
X, Y = mnist_reader.load_mnist(path=DATA_DIR, kind='t10k')
labels = ['t_shirt_top', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt', 'sneaker', 'bag', 'ankle_boots']
Y_str = np.array([labels[j] for j in Y])
np.savetxt('Xtest.tsv', X, fmt='%.6e', delimiter='\t')
np.savetxt('Ytest.tsv', Y_str, fmt='%s')
plt.imsave('zalando-mnist-sprite.png', get_sprite_image(X), cmap='gray')
exit()
embedding_var = tf.Variable(X, name='mnist_pixels')
# Format: tensorflow/tensorboard/plugins/projector/projector_config.proto
config = projector.ProjectorConfig()
# You can add multiple embeddings. Here we add only one.
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = VIS_DIR + 'Ytest.tsv'
embedding.sprite.image_path = VIS_DIR + 'zalando-mnist-sprite.png'
# Specify the width and height of a single thumbnail.
