def load_results(filenames):
assert len(filenames) > 0
data = dataset.load_cached(filenames[0])
for i in range(len(filenames) - 1):
data = dataset.combine(data, dataset.load_cached(filenames[i]))
data = dataset.DataSet(
features=data.features,
normalized=data.normalized.applymap(lambda x: x
if x > 0.9 else 0),
values=data.values,
)
return data
def load():
"""
Load the data-set into memory.
This uses a cache-file which is reloaded if it already exists,
otherwise the data-set is created and saved to
the cache-file. The reason for using a cache-file is that it
ensure the files are ordered consistently each time the data-set
is loaded. This is important when the data-set is used in
combination with Transfer Learning as is done in Tutorial #09.
:return:
A DataSet-object for the data-set.
"""
# Path for the cache-file.
cache_path = os.path.join(data_dir, dataset_name + ".pkl")
# If the DataSet-object already exists in a cache-file
# then load it, otherwise create a new object and save
# it to the cache-file so it can be loaded the next time.
dataset = load_cached(cache_path=cache_path,
in_dir=data_dir)
return dataset
def __init__(self, in_dir, save_folder=None):
dataset = load_cached(cache_path='my_dataset_cache.pkl', in_dir=in_dir)
self.num_classes = dataset.num_classes
image_paths_train, cls_train, self.labels_train = dataset.get_training_set(
)
image_paths_test, self.cls_test, self.labels_test = dataset.get_test_set(
)
##############################IMAGE PARAMETERS#####################################
self.img_size = 128
self.num_channels = 3
self.train_batch_size = 64
self.test_batch_size = 64
###################################################################################
self.x = tf.placeholder(
tf.float32,
shape=[None, self.img_size, self.img_size, self.num_channels],
name='x')
self.x_image = tf.reshape(
self.x, [-1, self.img_size, self.img_size, self.num_channels])
self.y_true = tf.placeholder(tf.float32,
shape=[None, self.num_classes],
name='y_true')
self.y_true_cls = tf.argmax(self.y_true, axis=1) #The True class Value
self.keep_prob = tf.placeholder(tf.float32)
self.keep_prob_2 = tf.placeholder(tf.float32)
self.y_pred_cls = None
self.train_images = self.load_images(image_paths_train)
self.test_images = self.load_images(image_paths_test)
self.save_folder = save_folder
self.optimizer, self.accuracy = self.define_model()
def load():
"""
Load the Knifey-Spoony data-set into memory.
This uses a cache-file which is reloaded if it already exists,
otherwise the Knifey-Spoony data-set is created and saved to
the cache-file. The reason for using a cache-file is that it
ensure the files are ordered consistently each time the data-set
is loaded. This is important when the data-set is used in
combination with Transfer Learning as is done in Tutorial #09.
:return:
A DataSet-object for the Knifey-Spoony data-set.
"""
# Path for the cache-file.
cache_path = os.path.join(data_dir, "knifey-spoony.pkl")
# If the DataSet-object already exists in a cache-file
# then load it, otherwise create a new object and save
# it to the cache-file so it can be loaded the next time.
dataset = load_cached(cache_path=cache_path,
in_dir=data_dir)
return dataset
def main():
parser = argparse.ArgumentParser()
parser.add_argument('filename', type=str)
parser.add_argument('--norm',
nargs='*',
default=['scale', 'cutoff', 'rawcutoff', 'sigmoid'])
parser.add_argument('--prune-out',
nargs='?',
type=argparse.FileType('w'),
default=sys.stdout)
parser.add_argument('--class-out',
nargs='?',
type=argparse.FileType('w'),
default=sys.stdout)
args = parser.parse_args()
rawdata = dataset.load_cached(args.filename)
print_PCA_components(rawdata)
train_dataset, test_dataset = split_dataset(rawdata)
for norm in args.norm:
data = normalize_data(train_dataset, norm)
compare_train(data, test_dataset, norm, args.prune_out)
_, get_feats = augment_features(data.features)
compare_classifiers(data, test_dataset, norm, get_feats,
args.class_out)
# Hack to remove HDBScan cache, as it doesn't know about different
# files or normalization schemes
models.reset_hdbscan_cache()
def load():
"""
Load the eye data-set into memory.
:return:
A DataSet-object for the eye data-set.
"""
# Path for the cache-file.
cache_path = os.path.join(data_dir, "eye.pkl")
# If the DataSet-object already exists in a cache-file
# then load it, otherwise create a new object and save
# it to the cache-file so it can be loaded the next time.
dataset = load_cached(cache_path=cache_path,
in_dir=data_dir)
return dataset
def __init__(self,in_dir,save_folder=None):
# dataset
dataset = dataset.load_cached(cache_path='gdrive/My Drive/Colab Notebooks/data/', in_dir=in_dir)
# number of classes
self.num_classes = dataset.num_classes
# get training set
image_paths_train, cls_train, self.labels_train = dataset.get_training_set()
# get test set
image_paths_test, self.cls_test, self.labels_test = dataset.get_test_set()
##############################IMAGE PARAMETERS#####################################
self.img_size = 128
self.num_channels = 3
# batch size
self.train_batch_size = 64
self.test_batch_size = 64
###################################################################################
# placeholder: setting the matrix(mapping)
# @params dtype: data type
# @params shape: shape of input data
# @params name: name of placeholder
# x: feature, x_imange: reshape of feature
self.x = tf.placeholder(tf.float32, shape=[None, self.img_size,self.img_size,self.num_channels], name='x')
self.x_image = tf.reshape(self.x, [-1, self.img_size, self.img_size, self.num_channels])
# placeholder: setting the matrix(mapping)
# y true value
self.y_true = tf.placeholder(tf.float32, shape=[None, self.num_classes], name='y_true')
# tf.argmax: return the max value
self.y_true_cls = tf.argmax(self.y_true, axis=1) #The True class Value
self.keep_prob = tf.placeholder(tf.float32)
self.keep_prob_2 = tf.placeholder(tf.float32)
self.y_pred_cls = None
# train images
self.train_images= self.load_images(image_paths_train)
# test images
self.test_images= self.load_images(image_paths_test)
self.save_folder=save_folder
self.optimizer,self.accuracy = self.define_model()
def load():
"""
Load the data-set into memory.
This uses a cache-file which is reloaded if it already exists,
otherwise the data-set is created and saved to
the cache-file. The reason for using a cache-file is that it
ensure the files are ordered consistently each time the data-set
is loaded.
:return:
A DataSet-object for the clouds data-set
Only four major cloud types used/considered
"""
# Path for the cache-file.
cache_path = os.path.join(data_dir, "clouds-images.pkl")
# If the DataSet-object already exists in a cache-file
# then load it, otherwise create a new object and save
# it to the cache-file so it can be loaded the next time.
dataset = load_cached(cache_path=cache_path, in_dir=data_dir)
return dataset
def __init__(self,in_dir,save_folder=None):
dataset = load_cached(cache_path='my_dataset_cache.pkl', in_dir=in_dir)
self.num_classes = dataset.num_classes
image_paths_train, cls_train, self.labels_train = dataset.get_training_set()
image_paths_test, self.cls_test, self.labels_test = dataset.get_test_set()
##############################IMAGE PARAMETERS#####################################
self.img_size = 128
self.num_channels = 3
self.train_batch_size = 64
self.test_batch_size = 64
###################################################################################
self.x = tf.placeholder(tf.float32, shape=[None, self.img_size,self.img_size,self.num_channels], name='x')
self.x_image = tf.reshape(self.x, [-1, self.img_size, self.img_size, self.num_channels])
self.y_true = tf.placeholder(tf.float32, shape=[None, self.num_classes], name='y_true')
self.y_true_cls = tf.argmax(self.y_true, axis=1) #The True class Value
self.keep_prob = tf.placeholder(tf.float32)
self.keep_prob_2 = tf.placeholder(tf.float32)
self.y_pred_cls = None
self.train_images= self.load_images(image_paths_train)
self.test_images= self.load_images(image_paths_test)
self.save_folder=save_folder
self.optimizer,self.accuracy = self.define_model()
def load_dataset(self):
dataset = load_cached(cache_path=self.pkl_file_name,
in_dir=self.dataset_directory)
self.image_paths_train, self.cls_train, self.labels_train = dataset.get_training_set(
)
def training_crossVal(kvalidation_splits=7,train_batch_size=2100,model_train=None,epochs=15,image_directory_path=None):
counter_epoch = 0
dataset = load_cached(cache_path='my_dataset_cache_repo.pkl',
in_dir=image_directory_path)
x_train, cls_train, y_train = dataset.get_training_set()
global image_paths_train=x_train
global labels_train= y_train
del x_train
del y_train
for i in range(epochs):
x, y = random_batch(train_batch_size=train_batch_size)
images_split = np.split(x, kvalidation_splits)
images_labels = np.split(y, kvalidation_splits)
del x
# for image_paths in images_split:
for count_i in range(kvalidation_splits):
image_paths = images_split[count_i]
train_image_label = images_labels[count_i]
train_image = np.empty((len(image_paths), 224, 224, 3))
# train_image=np.array([[]])
for i in range(len(image_paths)):
image = cv2.imread(image_paths[i])
if (image is not None):
resized_image = cv2.resize(image, dsize=(224, 224))
# resized_image_float=im2double(resized_image)
np_image = np.reshape(resized_image, (224, 224, 3))
np_image = np_image.astype('float32')
train_image[i] = np_image
else:
np.delete(image_paths, (i), axis=0)
np.delete(train_image, (i), axis=0)
np.delete(train_image_label, (i), axis=0)
# train_image=np.append(train_image,np_image)
del image, resized_image, np_image, image_paths
if count_i == kvalidation_splits - 1:
image_paths_val = images_split[0]
val_image_label = images_labels[0]
val_image = np.empty((len(image_paths_val), 224, 224, 3))
else:
image_paths_val = images_split[count_i + 1]
y_paths_val = images_labels[count_i + 1]
val_image = np.empty((len(image_paths_val), 224, 224, 3))
for j in range(len(image_paths_val)):
image = cv2.imread(image_paths_val[j])
if (image is not None):
resized_image = cv2.resize(image, dsize=(224, 224))
# resized_image_float=im2double(resized_image)
np_image = np.reshape(resized_image, (224, 224, 3))
np_image = np_image.astype('float32')
val_image[j] = np_image
else:
np.delete(image_paths_val, (j), axis=0)
np.delete(val_image, (j), axis=0)
np.delete(y_paths_val, (i), axis=0)
# np.save('images.npy',train_image_np)
del image, resized_image, np_image, image_paths_val
model_train.fit(x=train_image, y=train_image_label, epochs=1, batch_size=1, callbacks=callbacks_list2,
validation_data=(val_image, y_paths_val))
counter_epoch = counter_epoch + 1
print('Total epochs=' + str(counter_epoch))
model_train.save('logs/autoencoder.h5')
del train_image
del val_image
del images_split
return model_train
import os
import numpy as np
import prettytensor as pt
import tensorflow as tf
from dataset import load_cached
dataset = load_cached(cache_path='veg.pkl', in_dir='./veg')
num_classes = dataset.num_classes
class_names = dataset.class_names
# Make tensorflow shut up
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import inception
inception.maybe_download()
model = inception.Inception()
transfer_len = model.transfer_len
x = tf.placeholder(tf.float32, shape=[None, transfer_len], name='x')
y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y_true')
y_true_cls = tf.argmax(y_true, dimension=1)
is_Training = False
x_pretty = pt.wrap(x)
with pt.defaults_scope(activation_fn=tf.nn.relu):
y_pred, loss = x_pretty. \
fully_connected(size=4096, name='layer_fc1'). \
fully_connected(size=2048, name='layer_fc2'). \
dropout(keep_prob=0.5, phase=is_Training). \
