def get_processed_dataset(): train_path = 'pp_cifar10_train.pkl' test_path = 'pp_cifar10_test.pkl' if os.path.exists(train_path) and os.path.exists(test_path): print 'loading preprocessed data' trainset = serial.load(train_path) testset = serial.load(test_path) else: print 'loading raw data...' trainset = cifar10.CIFAR10(which_set="train") testset = cifar10.CIFAR10(which_set="test") pipeline = preprocessing.Pipeline() pipeline.items.append(preprocessing.ExtractPatchesWithPosition(patch_shape=patch_shape, patches_per_image=patches_per_image)) pipeline.items.append(preprocessing.GlobalContrastNormalization(sqrt_bias=10., use_std=True)) pipeline.items.append(preprocessing.PCA(num_components = num_components, keep_var_fraction = keep_var_fraction)) pipeline.items.append(preprocessing.ExtractPatchPairs(patches_per_image = patches_per_image, num_images = train_size, input_width = input_width)) trainset.apply_preprocessor(preprocessor=pipeline, can_fit=True) # the pkl-ing is having issues, the dataset is maybe too big. serial.save('pp_cifar10_train.pkl', trainset) serial.save('pp_cifar10_test.pkl', testset) # this path will be used for visualizing weights after training is done trainset.yaml_src = '!pkl: "%s"' % train_path testset.yaml_src = '!pkl: "%s"' % test_path return trainset, testset
def _load_data(self, which_set, context_len, data_mode): if data_mode not in ['words', 'chars']: raise ValueError("Only 'words' and 'chars' are possible values" "for data_mode, not %s" % (data_mode,)) path = "${PYLEARN2_DATA_PATH}/PennTreebankCorpus/" npz_data = serial.load(path + "penntree_char_and_word.npz") if which_set == 'train': self._raw_data = npz_data['train_' + data_mode] elif which_set == 'valid': self._raw_data = npz_data['valid_' + data_mode] elif which_set == 'test': self._raw_data = npz_data['test_' + data_mode] else: raise ValueError("Dataset must be one of 'train', 'valid' " "or 'test'") # Use word.lower() because the dictionary contains a single word # that is capitalized for some reason: N npz_data = serial.load(path + "dictionaries.npz") self._vocabulary = dict((word.lower(), word_index) for word_index, word in enumerate(npz_data['unique_' + data_mode])) if data_mode == 'words': self._unknown_index = 591 self._max_labels = 10000 else: self._unknown_index = 50 self._max_labels = 51 self._is_case_sensitive = False
def get_all_datasets(tot, preprocessors): for ii, preprocessor in enumerate(preprocessors): train_path = DATA_DIR+'train_'+preprocessor+'_preprocessed.pkl' valid_path = DATA_DIR+'valid_'+preprocessor+'_preprocessed.pkl' tottrain_path = DATA_DIR+'tottrain_'+preprocessor+'_preprocessed.pkl' test_path = DATA_DIR+'test_'+preprocessor+'_preprocessed.pkl' if not os.path.exists(train_path) or not os.path.exists(valid_path) or not os.path.exists(test_path): print('I cannot find something related to preprocessor: ' + preprocessor) else: if tot: trainset = serial.load(tottrain_path) else: trainset = serial.load(train_path) validset = serial.load(valid_path) testset = serial.load(test_path) if ii==0: tottrainset = trainset totvalidset = validset tottestset = testset else: tottrainset.X = np.append(tottrainset.X, trainset.X, axis=0) tottrainset.y = np.append(tottrainset.y, trainset.y, axis=0) return tottrainset, totvalidset, tottestset
def __init__(self,model): self.filename = model['filename'] try: try: self.model = serial.load(self.filename) except: self.filename = os.environ['MMDAEdaes'] + self.filename self.model = serial.load(self.filename) except Exception as e: print("error loading {}:".format(self.filename)) print(e) return False if not hasattr(self.model, 'sequence'): self.__extract_sequence() if not hasattr(self.model, 'mean') or not hasattr(self.model, 'std') or not hasattr(self.model, 'normalise'): self.__calc_mean_std() if not hasattr(self.model, 'function'): self.__create_function() if not hasattr(self.model, 'mmdae_type'): self.__determine_type() self.dtype = theano.config.floatX self.__clean()
def load_data(which="original", center=False, scale=False): if which == "original": path = "/data/lisa/data/faces/GoogleDataset/Clean/latest.pkl" data = serial.load(path) data_x = data[0] data_y = data[1] assert len(data_x) == len(data_y) if center: data_x -= 0.5 elif which == "kaggle": path = "/data/lisa/data/faces/EmotiW/preproc/samira/KGL-AFEW/" data_x = serial.load(path + "train_kaggle_x.npy") data_y = serial.load(path + "train_kaggle_y.npy") assert len(data_x) == len(data_y) if scale: data_x /= 255.0 if center: data_x -= 0.5 elif center: data_x -= 127.5 one_hot = np.zeros((data_y.shape[0], 7), dtype="float32") for i in xrange(data_y.shape[0]): one_hot[i, data_y[i]] = 1.0 data_y = one_hot return data_x.reshape(data_x.shape[0], 48 * 48).astype("float32"), data_y
def __call__(self): print 'loading model' if self.num_filters == 1600: d = serial.load('${USERDIR}/galatea/s3c/sc_vq_demo/omp1.mat') elif self.num_filters == 800: d = serial.load('/RQexec/goodfell/omp1_800.mat') else: assert False self.W = sharedX(d['dictionary'].T) self.size = int(np.sqrt(self.W.get_value().shape[0]/3)) if self.chunk_size is not None: dataset_family = self.dataset_family which_set = self.which_set dataset_descriptor = self.dataset_family[which_set][size] num_examples = dataset_descriptor.num_examples assert num_examples % self.chunk_size == 0 self.chunk_id = 0 for i in xrange(0,num_examples, self.chunk_size): self.restrict = (i, i + self.chunk_size) self._execute() self.chunk_id += 1 else: self._execute()
def train_layer4(supervised=True): global unsup_dataset, sup_dataset # Process unsupervised layer 4 unsup_dataset = TransformerDataset(raw=unsup_dataset, transformer=serial.load(layer3_unsup_model)) model = DenoisingAutoencoder(BinomialCorruptor(corruption_level=0.002), nvis=nhid3, nhid=nhid4, act_enc='tanh', act_dec=None, irange=0.5) training_alg = SGD(cost=MeanSquaredReconstructionError(), learning_rate=1e-4, batch_size= batch_size, monitoring_dataset=unsup_dataset, termination_criterion=EpochCounter(max_epochs=max_epochs)) extensions = [MonitorBasedLRAdjuster()] experiment = Train(dataset=unsup_dataset, model=model, algorithm=training_alg, save_path=layer4_unsup_model, save_freq=50, allow_overwrite=True, extensions=extensions) experiment.main_loop() if supervised: # Process supervised layer 4 layers = [PretrainedLayer(layer_name='h1', layer_content=serial.load(layer1_unsup_model), freeze_params=False), PretrainedLayer(layer_name='h2', layer_content=serial.load(layer2_unsup_model), freeze_params=False), PretrainedLayer(layer_name='h3', layer_content=serial.load(layer3_unsup_model), freeze_params=False), PretrainedLayer(layer_name='h4', layer_content=serial.load(layer4_unsup_model), freeze_params=False), Softmax(n_classes=class_number, layer_name='y', irange=0.5)] model = MLP(layers=layers, batch_size=sup_dataset.y.shape[0], nvis=nvis, layer_name=None) training_alg = SGD(learning_rate=1e-3, monitoring_dataset=sup_dataset, termination_criterion=EpochCounter(max_epochs=max_epochs_mlp)) experiment = Train(dataset=sup_dataset, model=model, algorithm=training_alg, save_path=layer4_sup_model, save_freq=50, allow_overwrite=True, extensions=extensions) experiment.main_loop() serial.save(layer1_unsup_model, model.layers[0].layer_content) serial.save(layer2_unsup_model, model.layers[1].layer_content) serial.save(layer3_unsup_model, model.layers[2].layer_content) serial.save(layer4_unsup_model, model.layers[3].layer_content)
def load_cifar10(): from pylearn2.utils import serial from pylearn2.datasets.zca_dataset import ZCA_Dataset # from pylearn2.datasets.cifar10 import CIFAR10 import theano def rotate_and_convert_grayscale(img): reshaped = img.reshape(32, 32, 3, order="F") rotated = np.rot90(reshaped, k=3) grayscaled = np.dot(rotated[:, :, :3], [0.299, 0.587, 0.144]) return grayscaled def transform(img_set): result = [] # Convert all images to grayscale and flatten the shape for img in img_set: # result.append(rotate_and_convert_grayscale(img).ravel()) result.append(img.ravel()) return np.array(result) # train_set = CIFAR10(which_set='train', start=0, stop=45000) # valid_set = CIFAR10(which_set='train', start=45000, stop=50000) # test_set = CIFAR10(which_set='test') data_path = os.getenv("PYLEARN2_DATA_PATH") whitened_path = os.path.join(data_path, "cifar10_cpu", "pylearn2_gcn_whitened") preprocessed_train_dataset = serial.load(os.path.join(whitened_path, "train.pkl")) preprocessed_test_dataset = serial.load(os.path.join(whitened_path, "test.pkl")) preprocesssor = serial.load(os.path.join(whitened_path, "preprocessor.pkl")) train_set = ZCA_Dataset(preprocessed_train_dataset, preprocesssor, start=0, stop=45000) valid_set = ZCA_Dataset(preprocessed_train_dataset, preprocesssor, start=45000, stop=50000) test_set = ZCA_Dataset(preprocessed_test_dataset, preprocesssor) # Convert the images to grayscale and flatten them train_set.X = transform(train_set.X) valid_set.X = transform(valid_set.X) test_set.X = transform(test_set.X) def shared_y_cast(y): shared_y = theano.shared(np.asarray(y, dtype=theano.config.floatX), borrow=True) return T.cast(shared_y, "int32") train_set_tuple = ( theano.shared(np.array(train_set.X, dtype=theano.config.floatX), borrow=True), shared_y_cast(train_set.y.ravel()), ) valid_set_tuple = ( theano.shared(np.array(valid_set.X, dtype=theano.config.floatX), borrow=True), shared_y_cast(valid_set.y.ravel()), ) test_set_tuple = ( theano.shared(np.array(test_set.X, dtype=theano.config.floatX), borrow=True), shared_y_cast(test_set.y.ravel()), ) return [train_set_tuple, valid_set_tuple, test_set_tuple]
def main(): args = check_argv() model = serial.load(args.model_fn) print "Constructing model" if model.__class__ == pylearn2.models.mlp.MLP: # This is a correspondence model print "Loaded:", args.model_fn if args.use_layer is not None: use_layer = args.use_layer print "Using encoding from layer", use_layer, "out of", len(model.layers) else: print "Using last layer out of", len(model.layers) use_layer = -1 dAEs = [l.layer_content for l in model.layers[:use_layer]] else : # This is a normal stacked dAE: get the other layers from filename assert args.use_layer is None, "layer already specified in filename" model_dir, basename = path.split(args.model_fn) use_layer = int(basename.split(".")[-2].replace("layer", "")) # if use_layer != 0: # This is not single-layer model dAEs = [] for layer in range(use_layer + 1): model_fn = path.join( model_dir, ".".join(basename.split(".")[:-2]) + ".layer" + str(layer) + ".pkl" ) print "Loading:", model_fn dAEs.append(serial.load(model_fn)) model = pylearn2.models.autoencoder.DeepComposedAutoencoder(dAEs) input_dataset = dict(serial.load(args.input_fn)) # Symbolic matrix of items to encode x = T.dmatrix('x') encoder = model.encode(x) encode_func = function([x], encoder) # Perform encoding print "Performing encoding" result = {} for (label, features) in input_dataset.items(): result[label] = encode_func(features) # Write encoded output input_basename = path.splitext(path.split(args.input_fn)[-1])[0] model_dir, model_basename = path.split(args.model_fn) model_basename = path.splitext(model_basename)[0] model_basename = path.split(model_dir)[-1] + "." + model_basename encoded_fn = path.join( output_dir, "encoded." + input_basename + "." + model_basename + (".layer" + str(args.use_layer) if args.use_layer is not None else "") + ".npz" ) print "Writing encoding:", encoded_fn np.savez(encoded_fn, **result)
def load_cifar10(cifar_path, confidence_ascending=None): from pylearn2.datasets.zca_dataset import ZCA_Dataset from pylearn2.utils import serial import theano import theano.tensor as T def flatten(img_set): result = [] for img in img_set: result.append(img.ravel()) return np.array(result) def shared_y_cast(y): shared_y = theano.shared(np.asarray(y, dtype=theano.config.floatX), borrow=True) return T.cast(shared_y, "int32") whitened_path = os.path.join(cifar_path, "pylearn2_gcn_whitened") preprocessed_train_dataset = serial.load(os.path.join(whitened_path, "train.pkl")) preprocessed_test_dataset = serial.load(os.path.join(whitened_path, "test.pkl")) preprocesssor = serial.load(os.path.join(whitened_path, "preprocessor.pkl")) train_set = ZCA_Dataset(preprocessed_train_dataset, preprocesssor, start=0, stop=45000) valid_set = ZCA_Dataset(preprocessed_train_dataset, preprocesssor, start=45000, stop=50000) test_set = ZCA_Dataset(preprocessed_test_dataset, preprocesssor) if confidence_ascending is not None: X_new = np.empty_like(train_set.X) y_new = np.empty_like(train_set.y) for i in range(len(X_new)): label = int(train_set.y[i]) index_new = confidence_ascending[label].pop(0) X_new[i] = train_set.X[index_new] y_new[i] = train_set.y[index_new] train_set.X = X_new train_set.y = y_new train_set.X = flatten(train_set.X) valid_set.X = flatten(valid_set.X) test_set.X = flatten(test_set.X) train_set_tuple = ( theano.shared(np.array(train_set.X, dtype=theano.config.floatX), borrow=True), shared_y_cast(train_set.y.ravel()), ) valid_set_tuple = ( theano.shared(np.array(valid_set.X, dtype=theano.config.floatX), borrow=True), shared_y_cast(valid_set.y.ravel()), ) test_set_tuple = ( theano.shared(np.array(test_set.X, dtype=theano.config.floatX), borrow=True), shared_y_cast(test_set.y.ravel()), ) return [train_set_tuple, valid_set_tuple, test_set_tuple]
def main(): data_dir = string.preprocess('${PYLEARN2_DATA_PATH}/stl10') print('Loading STL10-10 unlabeled and train datasets...') downsampled_dir = data_dir + '/stl10_32x32' data = serial.load(downsampled_dir + '/unlabeled.pkl') supplement = serial.load(downsampled_dir + '/train.pkl') print('Concatenating datasets...') data.set_design_matrix(np.concatenate((data.X, supplement.X), axis=0)) del supplement print("Preparing output directory...") patch_dir = data_dir + '/stl10_patches_8x8' serial.mkdir(patch_dir) README = open(patch_dir + '/README', 'w') README.write(textwrap.dedent(""" The .pkl files in this directory may be opened in python using cPickle, pickle, or pylearn2.serial.load. data.pkl contains a pylearn2 Dataset object defining an unlabeled dataset of 2 million 6x6 approximately whitened, contrast-normalized patches drawn uniformly at random from a downsampled (to 32x32) version of the STL-10 train and unlabeled datasets. preprocessor.pkl contains a pylearn2 Pipeline object that was used to extract the patches and approximately whiten / contrast normalize them. This object is necessary when extracting features for supervised learning or test set classification, because the extracted features must be computed using inputs that have been whitened with the ZCA matrix learned and stored by this Pipeline. They were created with the pylearn2 script make_stl10_patches.py. All other files in this directory, including this README, were created by the same script and are necessary for the other files to function correctly. """)) README.close() print("Preprocessing the data...") pipeline = preprocessing.Pipeline() pipeline.items.append(preprocessing.ExtractPatches(patch_shape=(8, 8), num_patches=2*1000*1000)) pipeline.items.append( preprocessing.GlobalContrastNormalization(sqrt_bias=10., use_std=True)) pipeline.items.append(preprocessing.ZCA()) data.apply_preprocessor(preprocessor=pipeline, can_fit=True) data.use_design_loc(patch_dir + '/data.npy') serial.save(patch_dir + '/data.pkl', data) serial.save(patch_dir + '/preprocessor.pkl', pipeline)
def __init__(self, soln_path, save_path, black_sheep_path): self.__dict__.update(locals()) del self.self soln = serial.load(soln_path) self.soln = soln.get_param_vector() black_sheep = serial.load(black_sheep_path) self.black_sheep = black_sheep.get_param_vector()
def train_yaml(yaml_file): train = yaml_parse.load(yaml_file) mode1 = serial.load(os.environ["MMDAErbms"] + "/laser_best.pkl") mode2 = serial.load(os.environ["MMDAErbms"] + "/command_best.pkl") deep = serial.load(os.environ["MMDAErbms"] + "/laser_command_best.pkl") models = [mode1, deep, deep, mode1, mode2] layers = list() f = theano.config.floatX for ii, layer in enumerate(train.model.layers): if type(layer) is FlattenerLayer: for l in layer.raw_layer.layers: layers.append(l) elif type(layer) is SplitterLayer: layers.append(layer.raw_layer) else: layers.append(layer) for ii, (layer, model) in enumerate(zip(layers, models)): if ii < len(layers) / 2: if type(layer) is Sigmoid: if layer.get_weights().shape != model.get_weights(): layer.set_weights(model.get_weights()[: layer.get_weights().shape[0], :].astype(f)) else: layer.set_weights(model.get_weights().astype(f)) if len(model.get_param_values()) == 4: layer.set_biases(model.get_param_values()[3].astype(f)) else: layer.set_biases(model.get_param_values()[2].astype(f)) else: if type(layer) is Sigmoid: if layer.enc_layer is None: layer.set_weights(model.get_weights().transpose().astype(f)) layer.set_biases(model.get_param_values()[0].astype(f)) elif type(layer) is LinearGaussian: params = model.get_param_values() if layer.enc_layer is None: layer.set_weights(params[2].transpose().astype(f)) layer.set_biases(params[1].astype(f)) beta = model.get_params()[0].eval() if isinstance(beta, N.ndarray): layer.beta.set_value(model.get_params()[0].eval().astype(f)) elif isinstance(beta, theano.sandbox.cuda.type.CudaNdarrayType): layer.beta.set_value(model.get_params()[0].eval().dtype(f)) del models del mode1 del mode2 del deep train.main_loop()
def load_from_numpy(self, filename_root, mmap_mode='r'): # Load the data inputs = serial.load(filename_root+'_inputs.npy') labels = serial.load(filename_root+'_labels.npy') # Quick checks to ensure a proper dataset has been loaded assert inputs.shape == (62000, 784) assert labels.shape[0] == inputs.shape[0] return inputs, labels
def extract_data(task_0, task_1): model = serial.load(task_0) num_params = num_parameters(model) valid_0 = model.monitor.channels['valid_y_misclass'].val_record[-1] model = serial.load(task_1) valid_1 = model.monitor.channels['valid_both_y_misclass'].val_record[-1] return num_params, float(valid_0), float(valid_1)
def _load_data(self, which_set, phone): """ Load the TIMIT data from disk. Parameters ---------- which_set : str Subset of the dataset to use (either "train", "valid" or "test") """ # Check which_set if which_set not in ['train', 'valid', 'test']: raise ValueError(which_set + " is not a recognized value. " + "Valid values are ['train', 'valid', 'test'].") # Create file paths timit_base_path = os.path.join(os.environ["PYLEARN2_DATA_PATH"], "timit/readable") speaker_info_list_path = os.path.join(timit_base_path, "spkrinfo.npy") phonemes_list_path = os.path.join(timit_base_path, "reduced_phonemes.pkl") words_list_path = os.path.join(timit_base_path, "words.pkl") speaker_features_list_path = os.path.join(timit_base_path, "spkr_feature_names.pkl") speaker_id_list_path = os.path.join(timit_base_path, "speakers_ids.pkl") if phone == 'full': raw_wav_path = os.path.join(timit_base_path, which_set + "_x_raw.npy") else: raw_wav_path = os.path.join("/u/kimtaeho/Documents/2_Project/speech_synthesis/code/datasets/",which_set + "_wav_"+phone+".npy") phonemes_path = os.path.join(timit_base_path, which_set + "_x_phonemes.npy") phones_path = os.path.join(timit_base_path, which_set + "_x_phones.npy") words_path = os.path.join(timit_base_path, which_set + "_x_words.npy") speaker_path = os.path.join(timit_base_path, which_set + "_spkr.npy") # Load data. For now most of it is not used, as only the acoustic # samples are provided, but this is bound to change eventually. # Global data if not self.audio_only: self.speaker_info_list = serial.load( speaker_info_list_path ).tolist().toarray() self.speaker_id_list = serial.load(speaker_id_list_path) self.speaker_features_list = serial.load(speaker_features_list_path) self.words_list = serial.load(words_list_path) self.phonemes_list = serial.load(phonemes_list_path) # Set-related data self.raw_wav = serial.load(raw_wav_path) if not self.audio_only: self.phonemes = serial.load(phonemes_path) self.phones = serial.load(phones_path) self.words = serial.load(words_path) self.speaker_id = numpy.asarray(serial.load(speaker_path), 'int')
def get_checkpoint(self): try: checkpoint = self.file_prefix + ".pkl" model = serial.load(checkpoint) except IOError: checkpoint = self.file_prefix + "_best.pkl" model = serial.load(checkpoint) except IOError: return None return model
def __init__(self, lat, lon): if not os.path.exists(MLP_FILE % (lat, lon)): raise OSError if not os.path.exists(LASSO_FILE % (lat, lon)): raise OSError self.mlp_dropout_model = serial.load(MLP_DROPOUT_FILE % (lat, lon)) self.mlp_model = serial.load(MLP_FILE % (lat, lon)) self.lasso_model = pickle.load(open(LASSO_FILE % (lat, lon), 'r')) self.test_data = load_data.load_supervised(1986, 1999, lat, lon, 50, which='test') self.lat = lat self.lon = lon
def __init__(self, lock, modelpath = "/home/vartiai6/Autoproject/4layermaxoutbest.mdl", preprocessorpath = "/home/vartiai6/Autoproject/4layermaxoutpreprocessorbest.pkl"): self.lock = lock self.modelpath = modelpath self.model = serial.load(modelpath) self.preprocessorpath = preprocessorpath self.preprocessor = serial.load(preprocessorpath) X = self.model.get_input_space().make_theano_batch() Y = self.model.fprop( X ) self.f = function( [X], Y )
def _load_batch_cifar10pre(dtype='float64'): """ load a batch in the CIFAR-10 format """ preproc = os.path.join(data_dir_cifar10pre, "preprocessor.pkl") preprocessor = serial.load(preproc) train = os.path.join(data_dir_cifar10pre, "train.pkl") train_set = ZCA_Dataset(preprocessed_dataset=serial.load(train), preprocessor = preprocessor, start=0, stop = 50000) test = os.path.join(data_dir_cifar10pre, "test.pkl") test_set = ZCA_Dataset(preprocessed_dataset= serial.load(test), preprocessor = preprocessor) return train_set, test_set
def _load_data(self, data_path, which_set): """ Load the TIMIT data from disk. Parameters ---------- which_set : str Subset of the dataset to use (either "train", "valid" or "test") """ # Check which_set if which_set not in ['train', 'valid', 'test']: raise ValueError(which_set + " is not a recognized value. " + "Valid values are ['train', 'valid', 'test'].") # Create file paths timit_base_path = os.path.join(data_path, "timit/readable") speaker_info_list_path = os.path.join(timit_base_path, "spkrinfo.npy") phonemes_list_path = os.path.join(timit_base_path, "reduced_phonemes.pkl") words_list_path = os.path.join(timit_base_path, "words.pkl") speaker_features_list_path = os.path.join(timit_base_path, "spkr_feature_names.pkl") speaker_id_list_path = os.path.join(timit_base_path, "speakers_ids.pkl") raw_wav_path = os.path.join(timit_base_path, which_set + "_x_raw.npy") phonemes_path = os.path.join(timit_base_path, which_set + "_x_phonemes.npy") phones_path = os.path.join(timit_base_path, which_set + "_x_phones.npy") words_path = os.path.join(timit_base_path, which_set + "_x_words.npy") speaker_path = os.path.join(timit_base_path, which_set + "_spkr.npy") # Load data. For now most of it is not used, as only the acoustic # samples are provided, but this is bound to change eventually. # Global data if not self.audio_only: self.speaker_info_list = serial.load( speaker_info_list_path ).tolist().toarray() self.speaker_id_list = serial.load(speaker_id_list_path) self.speaker_features_list = serial.load(speaker_features_list_path) self.words_list = serial.load(words_list_path) self.phonemes_list = serial.load(phonemes_list_path) # Set-related data self.raw_wav = serial.load(raw_wav_path) if not self.audio_only: self.phonemes = serial.load(phonemes_path) self.phones = serial.load(phones_path) self.words = serial.load(words_path) self.speaker_id = np.asarray(serial.load(speaker_path), 'int')
def get_labels_and_fold_indices(cifar10, cifar100, stl10): assert stl10 or cifar10 or cifar100 assert stl10+cifar10+cifar100 == 1 if stl10: print('loading entire stl-10 train set just to get the labels and folds') stl10 = serial.load("${PYLEARN2_DATA_PATH}/stl10/stl10_32x32/train.pkl") train_y = stl10.y fold_indices = stl10.fold_indices elif cifar10 or cifar100: if cifar10: print('loading entire cifar10 train set just to get the labels') cifar = CIFAR10(which_set = 'train') else: assert cifar100 print('loading entire cifar100 train set just to get the labels') cifar = CIFAR100(which_set = 'train') cifar.y = cifar.y_fine train_y = cifar.y assert train_y is not None fold_indices = np.zeros((5,40000),dtype='uint16') idx_list = np.cast['uint16'](np.arange(1,50001)) #mimic matlab format of stl10 for i in xrange(5): mask = idx_list < i * 10000 + 1 mask += idx_list >= (i+1) * 10000 + 1 fold_indices[i,:] = idx_list[mask] assert fold_indices.min() == 1 assert fold_indices.max() == 50000 return train_y, fold_indices
def validate(model_path): from pylearn2.utils import serial try: model = serial.load(model_path) except Exception, e: print model_path + "doesn't seem to be a valid model path, I got this error when trying to load it: " print e
def get_fprop_fn(variable_shape=False, include_pool=True): """ build a theano function that use SAE weights to get convolved(or pooled if include_pool is True) features from a given input """ conf = utils.get_config() paths = utils.get_paths() ae = serial.load(paths['sae']['model']) cnn_layer = 'cnn_layer_%i' % (conf['cnn_layers']) batch_size = conf[cnn_layer]['batch_size'] nhid = conf['sae']['nhid'] patch_size = conf['patch_size'] region_size = conf['region_size'] input = T.tensor4('input') filter_shape = (nhid, 1, patch_size, patch_size) filters = theano.shared(ae.get_weights().T.reshape(filter_shape)) if variable_shape: out = conv.conv2d(input, filters) else: image_shape = [batch_size, 1, region_size, region_size] out = conv.conv2d(input, filters, filter_shape=filter_shape, image_shape=image_shape) if include_pool: pool_fn = getattr(out, conf['pool_fn']) out = pool_fn(axis=(2, 3)) return theano.function([input], out)
def __init__(self, which_set, center=False, one_hot=False): path = "${PYLEARN2_DATA_PATH}/mnist/mnist_rotation_back_image/" \ + which_set obj = serial.load(path) X = obj['data'] X = N.cast['float32'](X) y = N.asarray(obj['labels']) self.one_hot = one_hot if one_hot: one_hot = N.zeros((y.shape[0], 10), dtype='float32') for i in xrange(y.shape[0]): one_hot[i, y[i]] = 1. y = one_hot if center: X -= X.mean(axis=0) view_converter = dense_design_matrix.DefaultViewConverter((28, 28, 1)) super(MNIST_rotated_background, self).__init__( X=X, y=y, view_converter=view_converter) assert not N.any(N.isnan(self.X))
def print_results(model_path, results_path): f = open(results_path, 'a+') model = serial.load(model_path) monitor = model.monitor channels = monitor.channels keys = ['train_y_misclass', 'valid_y_misclass', 'test_y_misclass'] for key in keys: print >>f, str(channels[key].val_record[-1]) + '\t', # added by XD datasets = ('test',) for which_set in datasets: predict = channels[which_set + '_y_' + 'predict'].val_record[-1] rain = channels[which_set + '_y_' + 'rain'].val_record[-1] predict_and_rain = channels[which_set + '_y_' + 'predict_and_rain'].val_record[-1] precision = predict_and_rain * 1. / predict recall = predict_and_rain * 1. / rain f1 = 2. * precision * recall / (precision + recall) print >>f, str(1. - precision) + '\t', print >>f, str(1. - recall) + '\t', print >>f, str(f1) + '\t', # print >>f, '\n' print >>f, model_path f.close()
def single_print_gz(filepath): errors = {} assert tarfile.is_tarfile(filepath) post = '_' + str(long(time.time())) path = filepath[:-3] # remove .gz dname = os.path.join(os.path.dirname(path), post) fname = os.path.basename(path) path = os.path.join(dname, fname) # cmd = "tar -xzvf {} -C {} {}".format(filepath, dname, fname) # print cmd # subprocess.call(cmd, shell=True) tar = tarfile.open(filepath) tar.extractall(path=dname) tar.close() assert isfile(path) model = serial.load(path) monitor = model.monitor del model gc.collect() channels = monitor.channels for key in keys: value = channels[key].val_record[-1] if isinstance(value, np.ndarray): value = value.min() print key, ':', value errors[key] = value cmd = 'rm -rf {}'.format(dname) print cmd subprocess.call(cmd, shell=True) return errors
def findActivations(model_name, listX_raw, which_layer, maxPixel): # 1. load model file import theano from pylearn2.utils import serial model = serial.load(model_name) print 'Model input space is ', model.get_input_space() # 2. find activations at that layer num_x = len(listX_raw) activation_list = [] for X in listX_raw: m = X.shape[0] X = np.reshape(X, (m,1,maxPixel, maxPixel)) X = np.swapaxes(X,1,2) X = np.swapaxes(X,2,3) print 'XReport type = {}. Dimension = {}'.format(type(X), np.shape(X)) activation = None batch_size = 100 for batchIndex in range(m/batch_size): _input = np.array(X[batchIndex*batch_size:(batchIndex+1)*batch_size], dtype=theano.config.floatX) fprop_results = model.fprop(theano.shared(_input, name='XReport'), return_all=True)[which_layer].eval() if batchIndex==0: print 'fprop_results shape', fprop_results.shape if activation is None: activation = fprop_results else: activation = np.concatenate((activation, fprop_results), axis=0) # need to flatten to be (num points, num features) print 'Activation shape before reshape', activation.shape activation = np.reshape(activation, (m,-1)) print 'After reshape', activation.shape activation_list.append(activation) return activation_list
def recordModelParams(model_name = "plankton_conv_visualize_model.pkl",numLayer = 3): model_path= trainedModelPath + model_name print 'Loading Model' model = serial.load(model_path) print "Done Loading Model" print "Input Space:\t", model.get_input_space() print "Target Space:\t", model.get_target_space() # same as get_output_space() print "Monitoring Data Specs", model.get_monitoring_data_specs() param_names = model.get_params() param_names = [tensorVar.name for tensorVar in param_names] print "Params Spec", param_names layer_names = [] for i in range(numLayer): strname = "c" + str(i) + "_b"; layer_names.append(strname) strname = "c" + str(i) + "_W"; layer_names.append(strname) # layer_names = ['c2_W', 'c2_b', 'c1_W', 'c1_b', 'c0_W', 'c0_b'] layer_names.reverse() print "type", type(param_names[0]) print "index of c0_W", param_names.index('c0_W') # assume there are 3 layers original_params = model.get_param_values() params_indices = [param_names.index(_name) for _name in layer_names] print "Parameter Indices for {} is {}".format(layer_names, params_indices) params = [original_params[_index] for _index in params_indices] cPickle.dump(params,open(model_path + ".params", "wb"))
def __call__(self): print 'loading model' model_path = self.model_path self.model = serial.load(model_path) self.model.set_dtype('float32') self.size = int(np.sqrt(self.model.nvis/3)) if self.chunk_size is not None: dataset_family = self.dataset_family which_set = self.which_set dataset_descriptor = self.dataset_family[which_set][size] num_examples = dataset_descriptor.num_examples assert num_examples % self.chunk_size == 0 self.chunk_id = 0 for i in xrange(0,num_examples, self.chunk_size): self.restrict = (i, i + self.chunk_size) self._execute() self.chunk_id += 1 else: self._execute()
# coding: UTF-8 import pickle # reader = pickle.load(open('result/convolutional_network_best.pkl', 'rb')) # => ImportError: Cuda not found. Cannot unpickle CudaNdarray from pylearn2.utils import serial import codecs def ccc(name): if name.lower() == 'windows-31j': return codecs.lookup('utf-8') codecs.register(ccc) data = serial.load('../data/mnist/mnist.pkl', 'rb') serial.save('../data/mnist/mnist_train_X.pkl', data[0][0]) serial.save('../data/mnist/mnist_train_y.pkl', data[0][1].reshape((-1, 1))) serial.save('../data/mnist/mnist_valid_X.pkl', data[1][0]) serial.save('../data/mnist/mnist_valid_y.pkl', data[1][1].reshape((-1, 1))) serial.save('../data/mnist/mnist_test_X.pkl', data[2][0]) serial.save('../data/mnist/mnist_test_y.pkl', data[2][1].reshape((-1, 1))) # 上記のdata[0][0]などの実体は、ndarray
def predict(model_path, test_path, output_path, predictionType="classification", outputType="int", headers=False, first_col_label=False, delimiter=","): """ Predict from a pkl file. Parameters ---------- modelFilename : str The file name of the model file. testFilename : str The file name of the file to test/predict. outputFilename : str The file name of the output file. predictionType : str, optional Type of prediction (classification/regression). outputType : str, optional Type of predicted variable (int/float). headers : bool, optional Indicates whether the first row in the input file is feature labels first_col_label : bool, optional Indicates whether the first column in the input file is row labels (e.g. row numbers) """ print("loading model...") try: model = serial.load(model_path) except Exception as e: print("error loading {}:".format(model_path)) print(e) return False print("setting up symbolic expressions...") X = model.get_input_space().make_theano_batch() Y = model.fprop(X) if predictionType == "classification": Y = T.argmax(Y, axis=1) f = function([X], Y, allow_input_downcast=True) print("loading data and predicting...") # x is a numpy array # x = pickle.load(open(test_path, 'rb')) skiprows = 1 if headers else 0 x = np.loadtxt(test_path, delimiter=delimiter, skiprows=skiprows) if first_col_label: x = x[:, 1:] y = f(x) print("writing predictions...") variableType = "%d" if outputType != "int": variableType = "%f" np.savetxt(output_path, y, fmt=variableType) return True
WRITEME """ #argument: path to a pkl file #loads the pkl file and figures out which fields are CudaNDArrays from __future__ import print_function import sys if __name__ == "__main__": path = sys.argv[1] from pylearn2.utils import serial import inspect obj = serial.load(path) from theano.sandbox.cuda import CudaNdarray visited = set([]) def find(cur_obj, cur_name): global visited if isinstance(cur_obj, CudaNdarray): print(cur_name) print(cur_name) for field, new_obj in inspect.getmembers(cur_obj): if new_obj in visited: continue
def __init__(self, which_set, fold=0, image_size=48, example_range=None, center=False, shuffle=False, rng=None, seed=132987): """ Creates a DenseDesignMatrix object for the Toronto Face Dataset. :param which_set: dataset to load. One of ['train','valid','test','unlabeled']. :param center: move data from range [0.,255.] to [-127.5,127.5] :param example_range: array_like. Load only examples in range [example_range[0]:example_range[1]]. :param fold: TFD contains 5 official folds for train, valid and test. :param image_size: one of [48,96]. Load smaller or larger dataset variant. """ assert which_set in self.mapper.keys() assert (fold >= 0) and (fold < 5) # load data path = '${PYLEARN2_DATA_PATH}/faces/TFD/' if image_size == 48: data = load(path + 'TFD_48x48.mat') elif image_size == 96: data = load(path + 'TFD_96x96.mat') else: raise ValueError("image_size should be either 48 or 96.") # retrieve indices corresponding to `which_set` and fold number set_indices = data['folds'][:, fold] == self.mapper[which_set] # limit examples returned to `example_range` ex_range = slice(example_range[0], example_range[1]) \ if example_range else slice(None) # get images and cast to float32 data_x = data['images'][set_indices] data_x = np.cast['float32'](data_x) data_x = data_x[ex_range] # create dense design matrix from topological view data_x = data_x.reshape(data_x.shape[0], image_size**2) if center: data_x -= 127.5 if shuffle: rng = rng if rng else np.random.RandomState(seed) rand_idx = rng.permutation(len(data_x)) data_x = data_x[rand_idx] # get labels if which_set != 'unlabeled': data_y = data['labs_ex'][set_indices] data_y = data_y[ex_range] if shuffle: data_y = data_y[rand_idx] else: data_y = None # create view converting for retrieving topological view view_converter = dense_design_matrix.DefaultViewConverter( (image_size, image_size, 1)) # init the super class super(TFD, self).__init__(X=data_x, y=data_y, view_converter=view_converter) assert not np.any(np.isnan(self.X))
def __init__(self, which_set, fold=0, image_size=48, example_range=None, center=False, scale=False, shuffle=False, one_hot=False, rng=None, seed=132987, preprocessor=None, axes=('b', 0, 1, 'c')): if which_set not in self.mapper.keys(): raise ValueError("Unrecognized which_set value: %s. Valid values" + "are %s." % (str(which_set), str(self.mapper.keys()))) assert (fold >= 0) and (fold < 5) self.args = locals() # load data path = '${PYLEARN2_DATA_PATH}/faces/TFD/' if image_size == 48: data = load(path + 'TFD_48x48.mat') elif image_size == 96: data = load(path + 'TFD_96x96.mat') else: raise ValueError("image_size should be either 48 or 96.") # retrieve indices corresponding to `which_set` and fold number if self.mapper[which_set] == 4: set_indices = (data['folds'][:, fold] == 1) + \ (data['folds'][:, fold] == 2) else: set_indices = data['folds'][:, fold] == self.mapper[which_set] assert set_indices.sum() > 0 # limit examples returned to `example_range` if example_range: ex_range = slice(example_range[0], example_range[1]) else: ex_range = slice(None) # get images and cast to float32 data_x = data['images'][set_indices] data_x = np.cast['float32'](data_x) data_x = data_x[ex_range] # create dense design matrix from topological view data_x = data_x.reshape(data_x.shape[0], image_size**2) if center and scale: data_x[:] -= 127.5 data_x[:] /= 127.5 elif center: data_x[:] -= 127.5 elif scale: data_x[:] /= 255. if shuffle: rng = make_np_rng(rng, seed, which_method='permutation') rand_idx = rng.permutation(len(data_x)) data_x = data_x[rand_idx] # get labels if which_set != 'unlabeled': data_y = data['labs_ex'][set_indices] data_y = data_y[ex_range] - 1 data_y_identity = data['labs_id'][set_indices] data_y_identity = data_y_identity[ex_range] if shuffle: data_y = data_y[rand_idx] data_y_identity = data_y_identity[rand_idx] self.one_hot = one_hot if one_hot: one_hot = np.zeros((data_y.shape[0], 7), dtype='float32') for i in xrange(data_y.shape[0]): one_hot[i, data_y[i]] = 1. data_y = one_hot else: data_y = None data_y_identity = None # create view converting for retrieving topological view view_converter = dense_design_matrix.DefaultViewConverter( (image_size, image_size, 1), axes) # init the super class super(TFD, self).__init__(X=data_x, y=data_y, view_converter=view_converter) assert not np.any(np.isnan(self.X)) self.y_identity = data_y_identity self.axes = axes if preprocessor is not None: preprocessor.apply(self)
For example, pkl_inspector.py foo.pkl .my_field [my_key] 3 will load an object obj from foo.pkl and analyze obj.my_field["my_key"][3] """ if __name__ == "__main__": if len(sys.argv) == 1: usage() sys.exit(-1) hp = pickle.HIGHEST_PROTOCOL filepath = sys.argv[1] orig_obj = serial.load(filepath) cycle_check = {} obj_name = 'root_obj' cycle_check[id(orig_obj)] = obj_name for field in sys.argv[2:]: if field.startswith('['): assert field.endswith(']') obj_name += '[' + field[1:-1] + ']' orig_obj = orig_obj[field[1:-1]] elif field.startswith('.'): obj_name += '.' + field orig_obj = getattr(orig_obj, field[1:]) else:
serial.save(components+'/whitener.pkl',whitener) serial.save(components+'/num_examples.pkl',num_examples) serial.save(components+'/expanded_dim.pkl',expanded_dim) print 'done, checking result' #checks from theano import function import theano.tensor as T pca_input = T.matrix() assert pca_input.dtype == floatX del whitener whitener = serial.load(components+'/whitener.pkl') out = whitener(pca_input) assert out.dtype == floatX out_func = function([pca_input],out) test = out_func((g1)) #print g1[0:5,0:5] """g1 -= whitener.mean.get_value() print 'after manual mean subtract, mean is' mu = g1.mean(axis=0) print (mu.min(), mu.max()) g1 = N.dot(g1,whitener.get_weights())
print "(You used the wrong # of arguments)" quit(-1) _, model_path, out_path, dataset_id = sys.argv import os if os.path.exists(out_path): usage() print out_path + " already exists, and I don't want to overwrite anything just to be safe." quit(-1) from pylearn2.utils import serial try: model = serial.load(model_path) except Exception, e: usage() print model_path + "doesn't seem to be a valid model path, I got this error when trying to load it: " print e from pylearn2.config import yaml_parse hps = KeypointHPS('make_submission', -1, None) row = hps.db.executeSQL( """ SELECT preprocess_array,train_ddm_id,valid_ddm_id,test_ddm_id FROM hps3.dataset WHERE dataset_id = %s """, (dataset_id, ), hps.db.FETCH_ONE) if not row or row is None:
if not args.which_layers: parser.error( 'Please specify --which_layers x, with x either 1, 2, 3 or 1 2 3 (layer 0 is a pre-processing layer)' ) if args.aggregate_features: print 'Using aggregate features' else: print 'Not using aggregate features' if args.classifier is None: print 'No classifer selected, using random forest' args.classifier = 'random_forest' # load model model = serial.load(args.model_file) # parse dataset from model p = re.compile(r"which_set.*'(train)'") trainset_yaml = model.dataset_yaml_src validset_yaml = p.sub("which_set: 'valid'", model.dataset_yaml_src) testset_yaml = p.sub("which_set: 'test'", model.dataset_yaml_src) trainset = yaml_parse.load(trainset_yaml) validset = yaml_parse.load(validset_yaml) testset = yaml_parse.load(testset_yaml) if args.aggregate_features: X_train, y_train, Z_train, train_files = aggregate_features( model, trainset, which_layers=args.which_layers) X_valid, y_valid, Z_valid, valid_files = aggregate_features(
import pickle import os from pylearn2.utils import serial import glob from os.path import basename daes = list() labels = list() for f in glob.glob(os.environ['MMDAEdaes'] + '*_best.pkl'): daes.append(serial.load(f)) labels.append(basename(f)) fo = open(os.environ['MMDAEdaes'] + 'daes.pkl','wb') pickle.dump([daes,labels], fo) fo.close()
from sklearn.externals.joblib import dump import sys from pylearn2.utils import serial model = serial.load(sys.argv[1]) all_weights = [] for l in model.generator.mlp.layers: all_weights.append(l.get_weights()) from IPython import embed; embed()
def load_dataset(which_set, dataset_types): # we need to have at least 2 types otherwise this func is useless assert len(dataset_types) > 1 print "loading.. ", which_set if which_set == 'test': start_set = 0 stop_set = 10000 elif which_set == 'valid': which_set = 'train' start_set = 40000 stop_set = 50000 else: #train start_set = 0 stop_set = 40000 n_classes = 10 data = [] for prepro in dataset_types: if prepro == 'gcn': print "LOADING GCN..." input_data = CIFAR10(which_set=which_set, start=start_set, stop=stop_set, gcn=55., axes=['b', 0, 1, 'c']) # gcn_data = input_data.get_topological_view() data.append(input_data.get_topological_view()) if prepro == 'toronto': print "LOADING TOR..." input_data = CIFAR10(which_set=which_set, start=start_set, stop=stop_set, axes=['b', 0, 1, 'c'], toronto_prepro=1) # tor_data = input_data.get_topological_view() data.append(input_data.get_topological_view()) if prepro == 'zca': print "LOADING ZCA..." data_dir = string_utils.preprocess('${PYLEARN2_DATA_PATH}/cifar10') input_data = ZCA_Dataset( preprocessed_dataset=serial.load(data_dir + "/pylearn2_gcn_whitened/" + which_set + ".pkl"), preprocessor=serial.load( data_dir + "/pylearn2_gcn_whitened/preprocessor.pkl"), start=start_set, stop=stop_set, axes=['b', 0, 1, 'c']) # zca_data = input_data.get_topological_view() data.append(input_data.get_topological_view()) target_data = OneHotFormatter(n_classes).format(input_data.y, mode="concatenate") data.append(target_data) data_source = [] for i in range(len(dataset_types)): data_source.append('features' + str(i)) data_source.append('targets') ################################## DEFINE SPACES ################################## spaces = [] # add input spaces as b01c for i in range(0, len(dataset_types)): spaces.append( Conv2DSpace(shape=(32, 32), num_channels=3, axes=('b', 0, 1, 'c'))) # add output space spaces.append(VectorSpace(n_classes)) set = VectorSpacesDataset(tuple(data), (CompositeSpace(spaces), tuple(data_source))) return set
#!/usr/bin/env python __authors__ = "Ian Goodfellow" __copyright__ = "Copyright 2010-2012, Universite de Montreal" __credits__ = ["Ian Goodfellow"] __license__ = "3-clause BSD" __maintainer__ = "Ian Goodfellow" __email__ = "goodfeli@iro" import numpy as np import sys path = sys.argv[1] from pylearn2.utils import serial model = serial.load(path) for param in model.get_params(): name = param.name if name is None: name = '<anon>' v = param.get_value() print name + ': ' + str((v.min(), v.mean(), v.max())) + ' ' + str(v.shape) if np.sign(v.min()) != np.sign(v.max()): v = np.abs(v) print 'abs(' + name + '): ' + str((v.min(), v.mean(), v.max())) if v.ndim == 2: row_norms = np.sqrt(np.square(v).sum(axis=1)) print name + " row norms: ", (row_norms.min(), row_norms.mean(), row_norms.max()) col_norms = np.sqrt(np.square(v).sum(axis=0)) print name + " col norms: ", (col_norms.min(), col_norms.mean(), col_norms.max()) if hasattr(model, 'monitor'):
job_name = 'cfa_olshausen' import numpy as N from pylearn2.pca import CovEigPCA from models import expand from pylearn2.utils import serial import time import SkyNet import gc print 'Loading MNIST train set' t1 = time.time() data = serial.load('olshausen.pkl') X = data.get_design_matrix() X = X[0:20000, :] t2 = time.time() print(t2 - t1), ' seconds' num_examples, input_dim = X.shape pca_dim = X.shape[1] print 'Training PCA with %d dimensions' % pca_dim t1 = time.time() pca_model = CovEigPCA(num_components=pca_dim) pca_model.train(X) pca_model.W.set_value(N.cast['float32'](N.identity(X.shape[1]))) pca_model.mean.set_value(N.cast['float32'](N.zeros(X.shape[1]))) t2 = time.time() print(t2 - t1), ' seconds' SkyNet.set_job_name(job_name)
It assumes that you have already run make_downsampled_stl10.py, which downsamples the STL-10 images to 1/3 of their original resolution. """ from pylearn2.utils import serial from pylearn2.datasets import preprocessing from pylearn2.utils import string import numpy as np data_dir = string.preprocess('${PYLEARN2_DATA_PATH}/stl10') print 'Loading STL-10 unlabeled and train datasets...' downsampled_dir = data_dir + '/stl10_32x32' data = serial.load(downsampled_dir + '/unlabeled.pkl') supplement = serial.load(downsampled_dir + '/train.pkl') print 'Concatenating datasets...' data.set_design_matrix(np.concatenate((data.X, supplement.X), axis=0)) print "Preparing output directory..." output_dir = data_dir + '/stl10_32x32_whitened' serial.mkdir(output_dir) README = open(output_dir + '/README', 'w') README.write(""" The .pkl files in this directory may be opened in python using cPickle, pickle, or pylearn2.serial.load. unsupervised.pkl, unlabeled.pkl, train.pkl, and test.pkl each contain
def _execute(self): batch_size = self.batch_size feature_type = self.feature_type pooling_region_counts = self.pooling_region_counts dataset_family = self.dataset_family which_set = self.which_set model = self.model size = self.size nan = 0 dataset_descriptor = dataset_family[which_set][size] dataset = dataset_descriptor.dataset_maker() expected_num_examples = dataset_descriptor.num_examples full_X = dataset.get_design_matrix() assert full_X.dtype == 'float32' num_examples = full_X.shape[0] assert num_examples == expected_num_examples if self.restrict is not None: assert self.restrict[1] <= full_X.shape[0] print 'restricting to examples ', self.restrict[ 0], ' through ', self.restrict[1], ' exclusive' full_X = full_X[self.restrict[0]:self.restrict[1], :] assert self.restrict[1] > self.restrict[0] #update for after restriction num_examples = full_X.shape[0] assert num_examples > 0 dataset.X = None dataset.design_loc = None dataset.compress = False patchifier = ExtractGridPatches(patch_shape=(size, size), patch_stride=(1, 1)) pipeline = serial.load(dataset_descriptor.pipeline_path) assert isinstance(pipeline.items[0], ExtractPatches) pipeline.items[0] = patchifier print 'defining features' V = T.matrix('V') assert V.type.dtype == 'float32' model.make_pseudoparams() d = model.infer(V=V) H = d['H_hat'] Mu1 = d['S_hat'] G = d['G_hat'] if len(G) != 1: raise NotImplementedError( "only supports two layer pd-dbms for now") G, = G assert H.dtype == 'float32' assert Mu1.dtype == 'float32' nfeat = model.s3c.nhid + model.dbm.rbms[0].nhid if self.feature_type == 'map_hs': feat = (H > 0.5) * Mu1 raise NotImplementedError("doesn't support layer 2") elif self.feature_type == 'map_h': feat = T.cast(H > 0.5, dtype='float32') raise NotImplementedError("doesn't support layer 2") elif self.feature_type == 'exp_hs': feat = H * Mu1 raise NotImplementedError("doesn't support layer 2") elif self.feature_type == 'exp_hs_split': Z = H * Mu1 pos = T.clip(Z, 0., 1e32) neg = T.clip(-Z, 0, 1e32) feat = T.concatenate((pos, neg), axis=1) nfeat *= 2 raise NotImplementedError("doesn't support layer 2") elif self.feature_type == 'exp_h,exp_g': feat = T.concatenate((H, G), axis=1) elif self.feature_type == 'exp_h_thresh': feat = H * (H > .01) raise NotImplementedError("doesn't support layer 2") else: raise NotImplementedError() assert feat.dtype == 'float32' print 'compiling theano function' f = function([V], feat) if config.device.startswith('gpu') and nfeat >= 4000: f = halver(f, nfeat) topo_feat_var = T.TensorType(broadcastable=(False, False, False, False), dtype='float32')() if self.pool_mode == 'mean': region_feat_var = topo_feat_var.mean(axis=(1, 2)) elif self.pool_mode == 'max': region_feat_var = topo_feat_var.max(axis=(1, 2)) else: raise ValueError("Unknown pool mode: " + self.pool_mode) region_features = function([topo_feat_var], region_feat_var) def average_pool(stride): def point(p): return p * ns / stride rval = np.zeros( (topo_feat.shape[0], stride, stride, topo_feat.shape[3]), dtype='float32') for i in xrange(stride): for j in xrange(stride): rval[:, i, j, :] = region_features( topo_feat[:, point(i):point(i + 1), point(j):point(j + 1), :]) return rval outputs = [ np.zeros((num_examples, count, count, nfeat), dtype='float32') for count in pooling_region_counts ] assert len(outputs) > 0 fd = DenseDesignMatrix(X=np.zeros((1, 1), dtype='float32'), view_converter=DefaultViewConverter( [1, 1, nfeat])) ns = 32 - size + 1 depatchifier = ReassembleGridPatches(orig_shape=(ns, ns), patch_shape=(1, 1)) if len(range(0, num_examples - batch_size + 1, batch_size)) <= 0: print num_examples print batch_size for i in xrange(0, num_examples - batch_size + 1, batch_size): print i t1 = time.time() d = copy.copy(dataset) d.set_design_matrix(full_X[i:i + batch_size, :]) t2 = time.time() #print '\tapplying preprocessor' d.apply_preprocessor(pipeline, can_fit=False) X2 = np.cast['float32'](d.get_design_matrix()) t3 = time.time() #print '\trunning theano function' feat = f(X2) t4 = time.time() assert feat.dtype == 'float32' feat_dataset = copy.copy(fd) if np.any(np.isnan(feat)): nan += np.isnan(feat).sum() feat[np.isnan(feat)] = 0 feat_dataset.set_design_matrix(feat) #print '\treassembling features' feat_dataset.apply_preprocessor(depatchifier) #print '\tmaking topological view' topo_feat = feat_dataset.get_topological_view() assert topo_feat.shape[0] == batch_size t5 = time.time() #average pooling for output, count in zip(outputs, pooling_region_counts): output[i:i + batch_size, ...] = average_pool(count) t6 = time.time() print(t6 - t1, t2 - t1, t3 - t2, t4 - t3, t5 - t4, t6 - t5) for output, save_path in zip(outputs, self.save_paths): if self.chunk_size is not None: assert save_path.endswith('.npy') save_path_pieces = save_path.split('.npy') assert len(save_path_pieces) == 2 assert save_path_pieces[1] == '' save_path = save_path_pieces[0] + '_' + chr( ord('A') + self.chunk_id) + '.npy' np.save(save_path, output) if nan > 0: warnings.warn(str(nan) + ' features were nan')
def __init__(self, models, datasets, normalise, which_set, batch_size, shuffle=False, start=None, stop=None, length=None, axes=['b', 0, 1, 'c']): self.args = locals() existing_data_path = os.environ['MMDAEdata'] + splitext( basename(models[0]))[0] + '_' + splitext(basename( datasets[0]))[0] + '_' + splitext( basename(models[1]))[0] + '_' + splitext( basename(datasets[1]))[0] + '_' + which_set + '.npy' assert which_set in ['train', 'valid'] assert type(models) is list assert type(datasets) is list assert len(models) == len(datasets) assert len(models) == len(normalise) def dimshuffle(b01c): default = ('b', 0, 1, 'c') return b01c.transpose(*[default.index(axis) for axis in axes]) # only process if it hasn't been done already if not os.path.exists(existing_data_path): for ii in range(len(models)): # Load single mode model for first layer model = serial.load(models[ii]) # Load the single mode data data = N.load(datasets[ii]) if normalise[ii] == 1: data_mean = data.mean() data_std = data.std() elif normalise[ii] == 2: data_mean = data.mean(axis=0) data_std = data.std(axis=0) data = (data - data_mean) / data_std sequence = model.dataset_yaml_src.split('sequence: ') if len(sequence) > 1: sequence = int(sequence[1].split(',')[0]) else: sequence = 1 if 'sequence_old' in locals(): assert sequence == sequence_old sequence_old = sequence if sequence != 1: temp = data data = np.zeros([temp.shape[0], sequence * temp.shape[1]]) for i in range(0, temp.shape[0] - sequence): data[i, :] = temp[i:i + sequence, :].reshape( 1, sequence * temp.shape[1]) del temp if start is not None: assert stop is not None assert start >= 0 assert stop > start assert ((stop - start) % batch_size) == 0 if stop > data.shape[0]: raise ValueError('stop=' + str(stop) + '>' + 'm=' + str(self.X.shape[0])) data = data[start:stop, :] if data.shape[0] != stop - start: raise ValueError("X.shape[0]: %d. start: %d stop: %d" % (self.X.shape[0], start, stop)) # Process data to get hidden representation from first layer data = theano.shared(data) data = model.mf(data) data = data[0] data = data[0] data = data.eval() if 'topo_view' not in locals(): topo_view = data.reshape(data.shape[0], 1, data.shape[1]) else: topo_view = N.append(topo_view, data.reshape(data.shape[0], 1, data.shape[1]), axis=2) m, r, c = topo_view.shape topo_view = topo_view.reshape(m, r, c, 1) # save the data to avoid reprocessing later N.save(existing_data_path, topo_view) else: topo_view = N.load(existing_data_path) m = topo_view.shape[0] if shuffle: self.shuffle_rng = make_np_rng(None, [1, 2, 3], which_method="shuffle") for i in xrange(topo_view.shape[0]): j = self.shuffle_rng.randint(m) # Copy ensures that memory is not aliased. tmp = topo_view[i, :, :, :].copy() topo_view[i, :, :, :] = topo_view[j, :, :, :] topo_view[j, :, :, :] = tmp super(CustomMMPosterior, self).__init__(topo_view=dimshuffle(topo_view)) assert not N.any(N.isnan(self.X))
# # print('Saving the unsupervised data') # train_set.use_design_loc(output_dir+'/train.npy') # serial.save(output_dir + '/train.pkl', train_set) # # print("Loading the test data") # test_set = CIFAR10(which_set='test') # # print("Preprocessing the test data") # test_set.apply_preprocessor(preprocessor=preprocessor, can_fit=False) # # print("Saving the test data") # test_set.use_design_loc(output_dir+'/test.npy') # serial.save(output_dir+'/test.pkl', test_set) train_set = serial.load(os.path.join(output_dir, 'train.pkl')) test_set = serial.load(os.path.join(output_dir, 'test.pkl')) preprocessor = serial.load(os.path.join(output_dir, 'preprocessor.pkl')) train_set = ZCA_Dataset(train_set, preprocessor, 0, 50000) test_set = ZCA_Dataset(test_set, preprocessor) train_set.X = train_set.X.reshape(-1, 3, 32, 32) test_set.X = test_set.X.reshape(-1, 3, 32, 32) # flatten targets train_set.y = np.hstack(train_set.y) test_set.y = np.hstack(test_set.y) # Onehot the targets
# Decaying LR LR_start = 0.003 print("LR_start = " + str(LR_start)) LR_fin = 0.000002 print("LR_fin = " + str(LR_fin)) LR_decay = (LR_fin / LR_start)**(1. / num_epochs) print("LR_decay = " + str(LR_decay)) # BTW, LR decay might good for the BN moving average... train_set_size = 45000 print("train_set_size = " + str(train_set_size)) print('Loading CIFAR-10 dataset...') preprocessor = serial.load( "${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/preprocessor.pkl") train_set = ZCA_Dataset(preprocessed_dataset=serial.load( "${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/train.pkl"), preprocessor=preprocessor, start=0, stop=train_set_size) valid_set = ZCA_Dataset(preprocessed_dataset=serial.load( "${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/train.pkl"), preprocessor=preprocessor, start=45000, stop=50000) test_set = ZCA_Dataset(preprocessed_dataset=serial.load( "${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/test.pkl"), preprocessor=preprocessor) # bc01 format
def __call__(self): return serial.load(self.path)
for seq in sequences: folded_seq = [fold_table.get(p, p) for p in seq] folded_sequences.append(folded_seq) return folded_sequences if __name__ == "__main__": parser = argparse.ArgumentParser( description= "Computes predictions of a model on TIMIT using phone folding.") parser.add_argument("model_file_name") args = parser.parse_args() timit_test = TIMIT('test', combine_stress=True) mlp = serial.load(args.model_file_name) Y_hat, Y_prob = get_model_predictions(mlp, timit_test) frame_classification_accuracy = np.all(Y_hat == timit_test.y, axis=1).mean() print "Frame classification error (raw): ", 1 - frame_classification_accuracy true_phone_sequences = get_phone_sequences(timit_test.y, timit_test.sentence_ids, timit_test.phone_index) predicted_phone_sequences = get_phone_sequences(Y_prob, timit_test.sentence_ids, timit_test.phone_index) print "Phone error rate (raw):", phone_error_rate( predicted_phone_sequences, true_phone_sequences)
def get_dataset(tot=False, preprocessor='normal'): if not os.path.exists(DATA_DIR+'train.npy') or \ not os.path.exists(DATA_DIR+'test.npy') or \ not os.path.exists(DATA_DIR+'targets.npy'): initial_read() train_path = DATA_DIR+'train_'+preprocessor+'_preprocessed.pkl' valid_path = DATA_DIR+'valid_'+preprocessor+'_preprocessed.pkl' tottrain_path = DATA_DIR+'tottrain_'+preprocessor+'_preprocessed.pkl' test_path = DATA_DIR+'test_'+preprocessor+'_preprocessed.pkl' if os.path.exists(train_path) and os.path.exists(valid_path) and os.path.exists(test_path): print 'loading preprocessed data' trainset = serial.load(train_path) validset = serial.load(valid_path) if tot: tottrainset = serial.load(tottrain_path) testset = serial.load(test_path) else: print 'loading raw data...' trainset = Digits(which_set='train', start=0, stop=34000) validset = Digits(which_set='train', start=34000, stop=42000) tottrainset = Digits(which_set='train') testset = Digits(which_set='test') print 'preprocessing data...' pipeline = preprocessing.Pipeline() pipeline.items.append(preprocessing.GlobalContrastNormalization(sqrt_bias=10., use_std=True)) if preprocessor != 'nozca': # ZCA = zero-phase component analysis # very similar to PCA, but preserves the look of the original image better pipeline.items.append(preprocessing.ZCA()) # note the can_fit=False's: no sharing between train and valid data trainset.apply_preprocessor(preprocessor=pipeline, can_fit=True) validset.apply_preprocessor(preprocessor=pipeline, can_fit=False) tottrainset.apply_preprocessor(preprocessor=pipeline, can_fit=True) testset.apply_preprocessor(preprocessor=pipeline, can_fit=False) if preprocessor not in ('normal','nozca'): for data in (trainset, validset, tottrainset, testset): for ii in range(data.X.shape[0]): # normalize to [0,1] dmax = np.max(data.X[ii,:]) dmin = np.min(data.X[ii,:]) dnorm = (data.X[ii,:] - dmin) / (dmax - dmin) # and convert to PIL image img = Image.fromarray(dnorm.reshape(28, 28) * 255.).convert('L') # apply preprocessor if preprocessor == 'rotate': rot = rng.randint(-40, 41) img = img.rotate(rot, Image.BILINEAR) elif preprocessor == 'emboss': img = emboss(img) elif preprocessor == 'hshear': # coef = 0 means unsheared coef = -1 + np.random.rand()*2 # note: image is moved with (coef/2)*28 to center it after shearing img = img.transform((28,28), Image.AFFINE, (1,coef,-(coef/2)*28,0,1,0), Image.BILINEAR) elif preprocessor == 'vshear': coef = -1 + np.random.rand()*2 img = img.transform((28,28), Image.AFFINE, (1,0,0,coef,1,-(coef/2)*28), Image.BILINEAR) elif preprocessor == 'patch': # negative values are not possible in PIL, so do a zoom only transform then x1 = np.random.randint(0, 5) y1 = np.random.randint(0, 5) x2 = np.random.randint(0, 5) y2 = np.random.randint(0, 5) img = img.transform((28,28), Image.EXTENT, (x1, y1, 28-x2, 28-y2), Image.BILINEAR) # convert back to numpy array data.X[ii,:] = np.array(img.getdata()) / 255. if preprocessor == 'noisy': # add noise data.X[ii,:] += np.random.randn(28*28) * 0.1 # bound between [0,1] data.X[ii,:] = np.minimum(np.ones(28*28), np.maximum(np.zeros(28*28), data.X[ii,:])) # this uses numpy format for storage instead of pickle, for memory reasons trainset.use_design_loc(DATA_DIR+'train_'+preprocessor+'_design.npy') validset.use_design_loc(DATA_DIR+'valid_'+preprocessor+'_design.npy') tottrainset.use_design_loc(DATA_DIR+'tottrain_'+preprocessor+'_design.npy') testset.use_design_loc(DATA_DIR+'test_'+preprocessor+'_design.npy') # this path can be used for visualizing weights after training is done trainset.yaml_src = '!pkl: "%s"' % train_path validset.yaml_src = '!pkl: "%s"' % valid_path tottrainset.yaml_src = '!pkl: "%s"' % tottrain_path testset.yaml_src = '!pkl: "%s"' % test_path print 'saving preprocessed data...' serial.save(train_path, trainset) serial.save(valid_path, validset) serial.save(tottrain_path, tottrainset) serial.save(test_path, testset) if tot: return tottrainset, validset, testset else: return trainset, validset, testset
def main(): trainset, validset, testset, extraset = get_dataset_icml() #trainset,testset = get_dataset_mnist() design_matrix = trainset.get_design_matrix() n_input = design_matrix.shape[1] n_output = 9 #10 # build layers layers = [] structure = [[n_input, 1000], [1000,1000],[1000,1000], [1000, n_output]] #layers.append(get_grbm(structure[0])) # layer 0: denoising AE layers.append(get_grbm(structure[0])) # layer 1: denoising AE layers.append(get_grbm(structure[1])) # layer 1: denoising AE layers.append(get_grbm(structure[2])) # layer 2: logistic regression used in supervised training #layers.append(get_logistic_regressor(structure[3])) #construct training sets for different layers traindata = [ extraset , TransformerDataset( raw = extraset, transformer = layers[0] ), TransformerDataset( raw = extraset, transformer = StackedBlocks( layers[0:2] )), TransformerDataset( raw = extraset, transformer = StackedBlocks( layers[0:3] )) ] #valid = TransformerDataset( raw = validset, transformer = StackedBlocks( layers[0:2] )) #valid = trainset # construct layer trainers layer_trainers = [] #layer_trainers.append(get_layer_trainer_sgd_rbm(layers[0], trainset[0])) layer_trainers.append(get_layer_trainer_sgd_rbm(layers[0], traindata[0],'db1.pkl')) layer_trainers.append(get_layer_trainer_sgd_rbm(layers[1], traindata[1],'db2.pkl')) layer_trainers.append(get_layer_trainer_sgd_rbm(layers[2], traindata[2],'db3.pkl')) #layer_trainers.append(get_layer_trainer_logistic(layers[2], trainset[2], valid)) #unsupervised pretraining for i, layer_trainer in enumerate(layer_trainers[0:3]): print '-----------------------------------' print ' Unsupervised training (pretraining) layer %d, %s'%(i, layers[i].__class__) print '-----------------------------------' layer_trainer.main_loop() print '\n' print '------------------------------------------------------' print ' Unsupervised training done! Start supervised training (fine-tuning)...' print '------------------------------------------------------' print '\n' mlp_layers = [] mlp_layers.append(PretrainedLayer(layer_name = 'h0', layer_content = serial.load('db1.pkl'))) mlp_layers.append(PretrainedLayer(layer_name = 'h1', layer_content = serial.load('db2.pkl'))) mlp_layers.append(PretrainedLayer(layer_name = 'h2', layer_content = serial.load('db3.pkl'))) #supervised training #layer_trainers[-1].main_loop() mlp_model = get_layer_MLP(mlp_layers,trainset,validset) mlp_model.main_loop()
randomize=[X_train, X_test], scale_diff=scale_diff, translation=translation, center_shape=center_shape, center=[X_train, X_test], preprocess=preprocess) train = Train(dataset=X_train, model=ann, algorithm=trainer, extensions=[watcher, velocity, decay, ra]) train.main_loop() print("using model", save_path) model = serial.load(save_path) print("loading test set") for f_name_dir in os.walk("test"): images_test, fnames, dims_test = load_images(f_name_dir, img_dim=img_dim, as_grey=as_grey) X_test = None p_test = np.zeros((len(images_test), 121), dtype=np.float32) for example in range(test_examples): print("creating test augmentation %d" % example) X_train = DenseDesignMatrix(X=images_train, y=y_train, view_converter=view_converter)
def get_weights_report(model_path=None, model=None, rescale='individual', border=False, norm_sort=False, dataset=None): """ Returns a PatchViewer displaying a grid of filter weights Parameters: model_path: the filepath of the model to make the report on. rescale: a string specifying how to rescale the filter images 'individual' (default): scale each filter so that it uses as much as possible of the dynamic range of the display under the constraint that 0 is gray and no value gets clipped 'global' : scale the whole ensemble of weights 'none' : don't rescale dataset: a Dataset object to do view conversion for displaying the weights. if not provided one will be loaded from the model's dataset_yaml_src """ if model is None: print 'making weights report' print 'loading model' model = serial.load(model_path) print 'loading done' else: assert model_path is None assert model is not None if rescale == 'none': global_rescale = False patch_rescale = False elif rescale == 'global': global_rescale = True patch_rescale = False elif rescale == 'individual': global_rescale = False patch_rescale = True else: raise ValueError('rescale=' + rescale + ", must be 'none', 'global', or 'individual'") if isinstance(model, dict): #assume this was a saved matlab dictionary del model['__version__'] del model['__header__'] del model['__globals__'] weights, = model.values() norms = np.sqrt(np.square(weights).sum(axis=1)) print 'min norm: ', norms.min() print 'mean norm: ', norms.mean() print 'max norm: ', norms.max() return patch_viewer.make_viewer(weights, is_color=weights.shape[1] % 3 == 0) weights_view = None W = None try: weights_view = model.get_weights_topo() h = weights_view.shape[0] except NotImplementedError: if dataset is None: print 'loading dataset...' control.push_load_data(False) dataset = yaml_parse.load(model.dataset_yaml_src) control.pop_load_data() print '...done' try: W = model.get_weights() except AttributeError, e: raise AttributeError(""" Encountered an AttributeError while trying to call get_weights on a model. This probably means you need to implement get_weights for this model class, but look at the original exception to be sure. If this is an older model class, it may have weights stored as weightsShared, etc. Original exception: """ + str(e))
def main(): """ .. todo:: WRITEME """ parser = argparse.ArgumentParser() parser.add_argument("--out") parser.add_argument("model_paths", nargs='+') options = parser.parse_args() model_paths = options.model_paths if options.out is not None: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt print 'generating names...' model_names = [ model_path.replace('.pkl', '!') for model_path in model_paths ] model_names = unique_substrings(model_names, min_size=10) model_names = [model_name.replace('!', '') for model_name in model_names] print '...done' for i, arg in enumerate(model_paths): try: model = serial.load(arg) except: if arg.endswith('.yaml'): print >> sys.stderr, arg + " is a yaml config file," + \ "you need to load a trained model." quit(-1) raise this_model_channels = model.monitor.channels if len(sys.argv) > 2: postfix = ":" + model_names[i] else: postfix = "" for channel in this_model_channels: channels[channel + postfix] = this_model_channels[channel] del model gc.collect() while True: # Make a list of short codes for each channel so user can specify them # easily tag_generator = _TagGenerator() codebook = {} sorted_codes = [] for channel_name in sorted(channels, key=number_aware_alphabetical_key): code = tag_generator.get_tag() codebook[code] = channel_name codebook['<' + channel_name + '>'] = channel_name sorted_codes.append(code) x_axis = 'example' print 'set x_axis to example' if len(channels.values()) == 0: print "there are no channels to plot" break # If there is more than one channel in the monitor ask which ones to # plot prompt = len(channels.values()) > 1 if prompt: # Display the codebook for code in sorted_codes: print code + '. ' + codebook[code] print print "Put e, b, s or h in the list somewhere to plot " + \ "epochs, batches, seconds, or hours, respectively." response = raw_input('Enter a list of channels to plot ' + \ '(example: A, C,F-G, h, <test_err>) or q to quit' + \ ' or o for options: ') if response == 'o': print '1: smooth all channels' print 'any other response: do nothing, go back to plotting' response = raw_input('Enter your choice: ') if response == '1': for channel in channels.values(): k = 5 new_val_record = [] for i in xrange(len(channel.val_record)): new_val = 0. count = 0. for j in xrange(max(0, i - k), i + 1): new_val += channel.val_record[j] count += 1. new_val_record.append(new_val / count) channel.val_record = new_val_record continue if response == 'q': break #Remove spaces response = response.replace(' ', '') #Split into list codes = response.split(',') final_codes = set([]) for code in codes: if code == 'e': x_axis = 'epoch' continue elif code == 'b': x_axis = 'batche' elif code == 's': x_axis = 'second' elif code == 'h': x_axis = 'hour' elif code.startswith('<'): assert code.endswith('>') final_codes.add(code) elif code.find('-') != -1: #The current list element is a range of codes rng = code.split('-') if len(rng) != 2: print "Input not understood: " + code quit(-1) found = False for i in xrange(len(sorted_codes)): if sorted_codes[i] == rng[0]: found = True break if not found: print "Invalid code: " + rng[0] quit(-1) found = False for j in xrange(i, len(sorted_codes)): if sorted_codes[j] == rng[1]: found = True break if not found: print "Invalid code: " + rng[1] quit(-1) final_codes = final_codes.union(set(sorted_codes[i:j + 1])) else: #The current list element is just a single code final_codes = final_codes.union(set([code])) # end for code in codes else: final_codes, = set(codebook.keys()) colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k'] styles = list(colors) styles += [color + '--' for color in colors] styles += [color + ':' for color in colors] fig = plt.figure() ax = plt.subplot(1, 1, 1) # plot the requested channels for idx, code in enumerate(sorted(final_codes)): channel_name = codebook[code] channel = channels[channel_name] y = np.asarray(channel.val_record) if np.any(np.isnan(y)): print channel_name + ' contains NaNs' if np.any(np.isinf(y)): print channel_name + 'contains infinite values' if x_axis == 'example': x = np.asarray(channel.example_record) elif x_axis == 'batche': x = np.asarray(channel.batch_record) elif x_axis == 'epoch': try: x = np.asarray(channel.epoch_record) except AttributeError: # older saved monitors won't have epoch_record x = np.arange(len(channel.batch_record)) elif x_axis == 'second': x = np.asarray(channel.time_record) elif x_axis == 'hour': x = np.asarray(channel.time_record) / 3600. else: assert False ax.plot( x, y, styles[idx % len(styles)], marker='.', # add point margers to lines label=channel_name) plt.xlabel('# ' + x_axis + 's') ax.ticklabel_format(scilimits=(-3, 3), axis='both') handles, labels = ax.get_legend_handles_labels() lgd = ax.legend(handles, labels, loc='upper center', bbox_to_anchor=(0.5, -0.1)) # 0.046 is the size of 1 legend box fig.subplots_adjust(bottom=0.11 + 0.046 * len(final_codes)) if options.out is None: plt.show() else: plt.savefig(options.out) if not prompt: break
def __init__(self, start_batch=0, stop_batch=782): self.__dict__.update(locals()) del self.self self.bow = serial.load('/data/lisatmp/goodfeli/esp_bow.pkl') self.global_rng = np.random.RandomState([2013, 3, 28]) self.y = self.bow.X[start_batch * 128:stop_batch * 128, :]
if not os.path.exists(expdir+'/validate_best.pkl'): print '\tExperiment done running but validate_best.pkl never showed up' print '\tLog file:',f print '\tValidation not yet run for experiment',expnum continue # we have to check if this exists, then load it # if we load it and respond to the exception, the exception is not specific enough # to know that the problem is the path not existing. if we check if the path exists # after we get the exception, the filesystem could have caught up while we're doing # the exception handling, and we'd decide that the exception must indicate a real # problem if not os.path.exists(expdir+'/validate.pkl'): print 'Experiment',expnum,' is suffering from the filesystem being stupid... validate_best.pkl exists but validate.pkl does not' continue model = serial.load(expdir+'/validate.pkl') monitor = model.monitor if not monitor.training_succeeded: print 'Training not done yet for experiment',expnum continue time = monitor.channels['valid_y_misclass'].time_record[-1] model = serial.load(expdir+'/validate_best.pkl') monitor = model.monitor obj = monitor.channels['valid_y_misclass'].val_record[-1] assert obj == min(monitor.channels['valid_y_misclass'].val_record) print expnum,obj,time
def __init__(self, which_set, center=False, rescale=False, gcn=None, start=None, stop=None, axes=('b', 0, 1, 'c'), toronto_prepro=False, preprocessor=None): # note: there is no such thing as the cifar10 validation set; # pylearn1 defined one but really it should be user-configurable # (as it is here) self.axes = axes # we define here: dtype = 'uint8' ntrain = 50000 nvalid = 0 # artefact, we won't use it ntest = 10000 # we also expose the following details: self.img_shape = (3, 32, 32) self.img_size = numpy.prod(self.img_shape) self.n_classes = 10 self.label_names = [ 'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck' ] # prepare loading fnames = ['data_batch_%i' % i for i in range(1, 6)] datasets = {} datapath = os.path.join( string_utils.preprocess('${PYLEARN2_DATA_PATH}'), 'cifar10', 'cifar-10-batches-py') for name in fnames + ['test_batch']: fname = os.path.join(datapath, name) if not os.path.exists(fname): raise IOError(fname + " was not found. You probably need to " "download the CIFAR-10 dataset by using the " "download script in " "pylearn2/scripts/datasets/download_cifar10.sh " "or manually from " "http://www.cs.utoronto.ca/~kriz/cifar.html") datasets[name] = cache.datasetCache.cache_file(fname) lenx = numpy.ceil((ntrain + nvalid) / 10000.) * 10000 x = numpy.zeros((lenx, self.img_size), dtype=dtype) y = numpy.zeros((lenx, 1), dtype=dtype) # load train data nloaded = 0 for i, fname in enumerate(fnames): _logger.info('loading file %s' % datasets[fname]) data = serial.load(datasets[fname]) x[i * 10000:(i + 1) * 10000, :] = data['data'] y[i * 10000:(i + 1) * 10000, 0] = data['labels'] nloaded += 10000 if nloaded >= ntrain + nvalid + ntest: break # load test data _logger.info('loading file %s' % datasets['test_batch']) data = serial.load(datasets['test_batch']) # process this data Xs = {'train': x[0:ntrain], 'test': data['data'][0:ntest]} Ys = {'train': y[0:ntrain], 'test': data['labels'][0:ntest]} X = numpy.cast['float32'](Xs[which_set]) y = Ys[which_set] if isinstance(y, list): y = numpy.asarray(y).astype(dtype) if which_set == 'test': assert y.shape[0] == 10000 y = y.reshape((y.shape[0], 1)) if center: X -= 127.5 self.center = center if rescale: X /= 127.5 self.rescale = rescale if toronto_prepro: assert not center assert not gcn X = X / 255. if which_set == 'test': other = CIFAR10(which_set='train') oX = other.X oX /= 255. X = X - oX.mean(axis=0) else: X = X - X.mean(axis=0) self.toronto_prepro = toronto_prepro self.gcn = gcn if gcn is not None: gcn = float(gcn) X = global_contrast_normalize(X, scale=gcn) if start is not None: # This needs to come after the prepro so that it doesn't # change the pixel means computed above for toronto_prepro assert start >= 0 assert stop > start assert stop <= X.shape[0] X = X[start:stop, :] y = y[start:stop, :] assert X.shape[0] == y.shape[0] if which_set == 'test': assert X.shape[0] == 10000 view_converter = dense_design_matrix.DefaultViewConverter((32, 32, 3), axes) super(CIFAR10, self).__init__(X=X, y=y, view_converter=view_converter, y_labels=self.n_classes) assert not contains_nan(self.X) if preprocessor: preprocessor.apply(self)
def __init__(self, which_set, shuffle=False, start=None, stop=None, axes=['b', 0, 1, 'c'], preprocessor=None, fit_preprocessor=False, fit_test_preprocessor=False): self.args = locals() if which_set not in ['train', 'valid', 'test']: raise ValueError('Unrecognized which_set value "%s".' % (which_set,) + '". Valid values are ' + '["train", "valid", "test"].') def dimshuffle(b01c): default = ('b', 0, 1, 'c') return b01c.transpose(*[default.index(axis) for axis in axes]) if control.get_load_data(): path = "${PYLEARN2_DATA_PATH}/binarized_mnist/binarized_mnist_" + \ which_set + ".npy" im_path = serial.preprocess(path) # Locally cache the files before reading them datasetCache = cache.datasetCache im_path = datasetCache.cache_file(im_path) try: X = serial.load(im_path) except IOError: raise NotInstalledError("BinarizedMNIST data files cannot be " "found in ${PYLEARN2_DATA_PATH}. Run " "pylearn2/scripts/datasets/" "download_binarized_mnist.py to get " "the data") else: if which_set == 'train': size = 50000 else: size = 10000 X = numpy.random.binomial(n=1, p=0.5, size=(size, 28 ** 2)) m, d = X.shape assert d == 28 ** 2 if which_set == 'train': assert m == 50000 else: assert m == 10000 if shuffle: self.shuffle_rng = make_np_rng(None, [1, 2, 3], which_method="shuffle") for i in xrange(X.shape[0]): j = self.shuffle_rng.randint(m) # Copy ensures that memory is not aliased. tmp = X[i, :].copy() X[i, :] = X[j, :] X[j, :] = tmp super(BinarizedMNIST, self).__init__( X=X, view_converter=DefaultViewConverter(shape=(28, 28, 1)) ) assert not numpy.any(numpy.isnan(self.X)) if start is not None: assert start >= 0 if stop > self.X.shape[0]: raise ValueError('stop=' + str(stop) + '>' + 'm=' + str(self.X.shape[0])) assert stop > start self.X = self.X[start:stop, :] if self.X.shape[0] != stop - start: raise ValueError("X.shape[0]: %d. start: %d stop: %d" % (self.X.shape[0], start, stop)) if which_set == 'test': assert fit_test_preprocessor is None or \ (fit_preprocessor == fit_test_preprocessor) if self.X is not None and preprocessor: preprocessor.apply(self, fit_preprocessor)