def load_results(filenames): assert len(filenames) > 0 data = dataset.load_cached(filenames[0]) for i in range(len(filenames) - 1): data = dataset.combine(data, dataset.load_cached(filenames[i])) data = dataset.DataSet( features=data.features, normalized=data.normalized.applymap(lambda x: x if x > 0.9 else 0), values=data.values, ) return data
def load(): """ Load the data-set into memory. This uses a cache-file which is reloaded if it already exists, otherwise the data-set is created and saved to the cache-file. The reason for using a cache-file is that it ensure the files are ordered consistently each time the data-set is loaded. This is important when the data-set is used in combination with Transfer Learning as is done in Tutorial #09. :return: A DataSet-object for the data-set. """ # Path for the cache-file. cache_path = os.path.join(data_dir, dataset_name + ".pkl") # If the DataSet-object already exists in a cache-file # then load it, otherwise create a new object and save # it to the cache-file so it can be loaded the next time. dataset = load_cached(cache_path=cache_path, in_dir=data_dir) return dataset
def __init__(self, in_dir, save_folder=None): dataset = load_cached(cache_path='my_dataset_cache.pkl', in_dir=in_dir) self.num_classes = dataset.num_classes image_paths_train, cls_train, self.labels_train = dataset.get_training_set( ) image_paths_test, self.cls_test, self.labels_test = dataset.get_test_set( ) ##############################IMAGE PARAMETERS##################################### self.img_size = 128 self.num_channels = 3 self.train_batch_size = 64 self.test_batch_size = 64 ################################################################################### self.x = tf.placeholder( tf.float32, shape=[None, self.img_size, self.img_size, self.num_channels], name='x') self.x_image = tf.reshape( self.x, [-1, self.img_size, self.img_size, self.num_channels]) self.y_true = tf.placeholder(tf.float32, shape=[None, self.num_classes], name='y_true') self.y_true_cls = tf.argmax(self.y_true, axis=1) #The True class Value self.keep_prob = tf.placeholder(tf.float32) self.keep_prob_2 = tf.placeholder(tf.float32) self.y_pred_cls = None self.train_images = self.load_images(image_paths_train) self.test_images = self.load_images(image_paths_test) self.save_folder = save_folder self.optimizer, self.accuracy = self.define_model()
def load(): """ Load the Knifey-Spoony data-set into memory. This uses a cache-file which is reloaded if it already exists, otherwise the Knifey-Spoony data-set is created and saved to the cache-file. The reason for using a cache-file is that it ensure the files are ordered consistently each time the data-set is loaded. This is important when the data-set is used in combination with Transfer Learning as is done in Tutorial #09. :return: A DataSet-object for the Knifey-Spoony data-set. """ # Path for the cache-file. cache_path = os.path.join(data_dir, "knifey-spoony.pkl") # If the DataSet-object already exists in a cache-file # then load it, otherwise create a new object and save # it to the cache-file so it can be loaded the next time. dataset = load_cached(cache_path=cache_path, in_dir=data_dir) return dataset
def main(): parser = argparse.ArgumentParser() parser.add_argument('filename', type=str) parser.add_argument('--norm', nargs='*', default=['scale', 'cutoff', 'rawcutoff', 'sigmoid']) parser.add_argument('--prune-out', nargs='?', type=argparse.FileType('w'), default=sys.stdout) parser.add_argument('--class-out', nargs='?', type=argparse.FileType('w'), default=sys.stdout) args = parser.parse_args() rawdata = dataset.load_cached(args.filename) print_PCA_components(rawdata) train_dataset, test_dataset = split_dataset(rawdata) for norm in args.norm: data = normalize_data(train_dataset, norm) compare_train(data, test_dataset, norm, args.prune_out) _, get_feats = augment_features(data.features) compare_classifiers(data, test_dataset, norm, get_feats, args.class_out) # Hack to remove HDBScan cache, as it doesn't know about different # files or normalization schemes models.reset_hdbscan_cache()
def load(): """ Load the eye data-set into memory. :return: A DataSet-object for the eye data-set. """ # Path for the cache-file. cache_path = os.path.join(data_dir, "eye.pkl") # If the DataSet-object already exists in a cache-file # then load it, otherwise create a new object and save # it to the cache-file so it can be loaded the next time. dataset = load_cached(cache_path=cache_path, in_dir=data_dir) return dataset
def __init__(self,in_dir,save_folder=None): # dataset dataset = dataset.load_cached(cache_path='gdrive/My Drive/Colab Notebooks/data/', in_dir=in_dir) # number of classes self.num_classes = dataset.num_classes # get training set image_paths_train, cls_train, self.labels_train = dataset.get_training_set() # get test set image_paths_test, self.cls_test, self.labels_test = dataset.get_test_set() ##############################IMAGE PARAMETERS##################################### self.img_size = 128 self.num_channels = 3 # batch size self.train_batch_size = 64 self.test_batch_size = 64 ################################################################################### # placeholder: setting the matrix(mapping) # @params dtype: data type # @params shape: shape of input data # @params name: name of placeholder # x: feature, x_imange: reshape of feature self.x = tf.placeholder(tf.float32, shape=[None, self.img_size,self.img_size,self.num_channels], name='x') self.x_image = tf.reshape(self.x, [-1, self.img_size, self.img_size, self.num_channels]) # placeholder: setting the matrix(mapping) # y true value self.y_true = tf.placeholder(tf.float32, shape=[None, self.num_classes], name='y_true') # tf.argmax: return the max value self.y_true_cls = tf.argmax(self.y_true, axis=1) #The True class Value self.keep_prob = tf.placeholder(tf.float32) self.keep_prob_2 = tf.placeholder(tf.float32) self.y_pred_cls = None # train images self.train_images= self.load_images(image_paths_train) # test images self.test_images= self.load_images(image_paths_test) self.save_folder=save_folder self.optimizer,self.accuracy = self.define_model()
def load(): """ Load the data-set into memory. This uses a cache-file which is reloaded if it already exists, otherwise the data-set is created and saved to the cache-file. The reason for using a cache-file is that it ensure the files are ordered consistently each time the data-set is loaded. :return: A DataSet-object for the clouds data-set Only four major cloud types used/considered """ # Path for the cache-file. cache_path = os.path.join(data_dir, "clouds-images.pkl") # If the DataSet-object already exists in a cache-file # then load it, otherwise create a new object and save # it to the cache-file so it can be loaded the next time. dataset = load_cached(cache_path=cache_path, in_dir=data_dir) return dataset
def __init__(self,in_dir,save_folder=None): dataset = load_cached(cache_path='my_dataset_cache.pkl', in_dir=in_dir) self.num_classes = dataset.num_classes image_paths_train, cls_train, self.labels_train = dataset.get_training_set() image_paths_test, self.cls_test, self.labels_test = dataset.get_test_set() ##############################IMAGE PARAMETERS##################################### self.img_size = 128 self.num_channels = 3 self.train_batch_size = 64 self.test_batch_size = 64 ################################################################################### self.x = tf.placeholder(tf.float32, shape=[None, self.img_size,self.img_size,self.num_channels], name='x') self.x_image = tf.reshape(self.x, [-1, self.img_size, self.img_size, self.num_channels]) self.y_true = tf.placeholder(tf.float32, shape=[None, self.num_classes], name='y_true') self.y_true_cls = tf.argmax(self.y_true, axis=1) #The True class Value self.keep_prob = tf.placeholder(tf.float32) self.keep_prob_2 = tf.placeholder(tf.float32) self.y_pred_cls = None self.train_images= self.load_images(image_paths_train) self.test_images= self.load_images(image_paths_test) self.save_folder=save_folder self.optimizer,self.accuracy = self.define_model()
def load_dataset(self): dataset = load_cached(cache_path=self.pkl_file_name, in_dir=self.dataset_directory) self.image_paths_train, self.cls_train, self.labels_train = dataset.get_training_set( )
def training_crossVal(kvalidation_splits=7,train_batch_size=2100,model_train=None,epochs=15,image_directory_path=None): counter_epoch = 0 dataset = load_cached(cache_path='my_dataset_cache_repo.pkl', in_dir=image_directory_path) x_train, cls_train, y_train = dataset.get_training_set() global image_paths_train=x_train global labels_train= y_train del x_train del y_train for i in range(epochs): x, y = random_batch(train_batch_size=train_batch_size) images_split = np.split(x, kvalidation_splits) images_labels = np.split(y, kvalidation_splits) del x # for image_paths in images_split: for count_i in range(kvalidation_splits): image_paths = images_split[count_i] train_image_label = images_labels[count_i] train_image = np.empty((len(image_paths), 224, 224, 3)) # train_image=np.array([[]]) for i in range(len(image_paths)): image = cv2.imread(image_paths[i]) if (image is not None): resized_image = cv2.resize(image, dsize=(224, 224)) # resized_image_float=im2double(resized_image) np_image = np.reshape(resized_image, (224, 224, 3)) np_image = np_image.astype('float32') train_image[i] = np_image else: np.delete(image_paths, (i), axis=0) np.delete(train_image, (i), axis=0) np.delete(train_image_label, (i), axis=0) # train_image=np.append(train_image,np_image) del image, resized_image, np_image, image_paths if count_i == kvalidation_splits - 1: image_paths_val = images_split[0] val_image_label = images_labels[0] val_image = np.empty((len(image_paths_val), 224, 224, 3)) else: image_paths_val = images_split[count_i + 1] y_paths_val = images_labels[count_i + 1] val_image = np.empty((len(image_paths_val), 224, 224, 3)) for j in range(len(image_paths_val)): image = cv2.imread(image_paths_val[j]) if (image is not None): resized_image = cv2.resize(image, dsize=(224, 224)) # resized_image_float=im2double(resized_image) np_image = np.reshape(resized_image, (224, 224, 3)) np_image = np_image.astype('float32') val_image[j] = np_image else: np.delete(image_paths_val, (j), axis=0) np.delete(val_image, (j), axis=0) np.delete(y_paths_val, (i), axis=0) # np.save('images.npy',train_image_np) del image, resized_image, np_image, image_paths_val model_train.fit(x=train_image, y=train_image_label, epochs=1, batch_size=1, callbacks=callbacks_list2, validation_data=(val_image, y_paths_val)) counter_epoch = counter_epoch + 1 print('Total epochs=' + str(counter_epoch)) model_train.save('logs/autoencoder.h5') del train_image del val_image del images_split return model_train
import os import numpy as np import prettytensor as pt import tensorflow as tf from dataset import load_cached dataset = load_cached(cache_path='veg.pkl', in_dir='./veg') num_classes = dataset.num_classes class_names = dataset.class_names # Make tensorflow shut up os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import inception inception.maybe_download() model = inception.Inception() transfer_len = model.transfer_len x = tf.placeholder(tf.float32, shape=[None, transfer_len], name='x') y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y_true') y_true_cls = tf.argmax(y_true, dimension=1) is_Training = False x_pretty = pt.wrap(x) with pt.defaults_scope(activation_fn=tf.nn.relu): y_pred, loss = x_pretty. \ fully_connected(size=4096, name='layer_fc1'). \ fully_connected(size=2048, name='layer_fc2'). \ dropout(keep_prob=0.5, phase=is_Training). \