def sat4_mc_test(label, num_corruptions, channel): """ Generates a dataset from single data sample corrupted multiple times.""" inFilename = './datasets/sat4-sat6/sat4_scaled_ZCA_retain_False_epsilon_0.001_rastered.npz' train_set, _, _ = load_data_npz(inFilename) # take first example of given label sample = train_set[0][train_set[1] == label][0] print 'taking sample shape as:{}'.format(sample.shape) corruptions = np.zeros((num_corruptions + 1, sample.shape[0], sample.shape[1], sample.shape[2])) scaling = 1.0 for i in xrange(num_corruptions + 1): print scaling temp = sample temp[channel] = temp[channel] * scaling corruptions[i] = temp scaling -= 0.1 print 'dataset shape is:{}'.format(corruptions.shape) outFilename = './datasets/sat4-sat6/sat4_label_{0}_corrupted_{1}_channel_{2}.npz'.format( label, num_corruptions, channel) with open(outFilename, 'wb') as f: np.savez(f, X_test=corruptions) f.close()
def main(argv): data_file = None results_dir = None try: opts, args = getopt.getopt(argv, "hd:r:", ["data_file=", "results_dir="]) except getopt.GetoptError: print 'incorrect usage' print 'usage: softmax.py -d <data_file> -r <results_dir>' sys.exit(2) for opt, arg in opts: if opt == "-h": print 'usage: softmax.py -d <data_file> -r <results_dir>' elif opt in ("-d", "--data_file"): data_file = arg elif opt in ("-r", "--results__dir"): results_dir = arg if data_file is None: print 'data_file was not given' print 'usage: softmax.py -d <data_file> -r <results_dir>' sys.exit(2) try: train_set, validation_set, test_set = load_data_npz(data_file) except IOError: print 'cannot open data_file', data_file X_train = train_set[0] X_train = toShared_x(X_train) X_test = test_set[0] X_test = toShared_x(X_test) nChannels = 4 dimHidden = 100 epochs = 15 miniBatchSize = 20 alpha = 0.001 nChains = 20 n_plotting_samples = 10 k = 15 persistent_bool = True apply_rbm_sgd(X_train, X_test, nChannels, dimHidden, epochs, miniBatchSize, alpha, nChains, n_plotting_samples, persistent_bool, k, results_dir)
def plot_channel_hists(filename, channel, class_labels, class_label_descriptions, zorders, colors, scale_to_unit_interval=False, rastered_input=False): """ plots histograms for each class in class_labels for channel channel """ train_set, _, _ = load_data_npz(filename) X_train = train_set[0] y_train = train_set[1] assert len(class_labels) == len(class_label_descriptions) == len( zorders) == len(colors) assert max(class_labels) <= y_train.max() and min( class_labels) >= y_train.min() if scale_to_unit_interval: X_train = unit_scale(X_train) if rastered_input: # X_train must be of shape [nSamples, nChannels, rows, cols] assert len(X_train.shape) == 4 spectral_intensities = np.squeeze( np.apply_over_axes(np.mean, X_train, [2, 3])) # average over raster xlabel = 'Average channel intensity (normalised)' else: spectral_intensities = X_train xlabel = 'Channel intensity (normalised)' plt.figure() for label, description, zorder, color in zip(class_labels, class_label_descriptions, zorders, colors): plt.hist(spectral_intensities[y_train == label, channel], bins=100, label=description, zorder=zorder, color=color) plt.xlim(0.0, 1.0) plt.xlabel(xlabel, fontsize=14) plt.ylabel('number of samples', fontsize=14) plt.legend() plt.show()
def main(argv): data_file = None results_dir = None try: opts, args = getopt.getopt(argv, "hd:r:", ["data_file=", "results_dir="]) except getopt.GetoptError: print 'incorrect usage' print 'usage: softmax.py -d <data_file> -r <results_dir>' sys.exit(2) for opt, arg in opts: if opt == "-h": print 'usage: softmax.py -d <data_file> -r <results_dir>' elif opt in ("-d", "--data_file"): data_file = arg elif opt in ("-r", "--results__dir"): results_dir = arg if data_file is None: print 'data_file was not given' print 'usage: softmax.py -d <data_file> -r <results_dir>' sys.exit(2) try: train_set, validation_set, test_set = load_data_npz(data_file) except IOError: print 'cannot open data_file', data_file X = train_set[0] X = toShared_x(X) nChannels = 4 dimHidden = 100 epochs = 15 miniBatchSize = 20 alpha = 0.001 momentum = None apply_dA_sgd(X, nChannels, dimHidden, epochs, miniBatchSize, alpha, results_dir, momentum)
def plot_covariance(filename, labels, rastered=False): train_set, _, _ = load_data_npz(filename) X_train = train_set[0].astype(float) if rastered: X_train = train_set[0].swapaxes(3, 1).reshape(-1, 4) assert len(labels) == X_train.shape[1] cov = np.dot(np.transpose(X_train), X_train) / X_train.shape[0] fig = plt.figure() ax = fig.add_subplot(111) cax = ax.matshow(cov, interpolation='nearest') fig.colorbar(cax) ax.set_xticklabels([''] + labels) ax.set_yticklabels([''] + labels) plt.show()
def plot_spec_profile(filename, class_label, channel_descriptions, zorders, colors, scale_to_unit_interval=False, rastered_input=False): """ plots spectral profile for class class_label """ train_set, _, _ = load_data_npz(filename) X_train = train_set[0] y_train = train_set[1] assert len(zorders) == len(colors) if scale_to_unit_interval: X_train = unit_scale(X_train) if rastered_input: # X_train must be of shape [nSamples, nChannels, rows, cols] assert len(X_train.shape) == 4 spectral_intensities = np.squeeze( np.apply_over_axes(np.mean, X_train, [2, 3])) # average over raster xlabel = 'Average channel intensity (normalised)' else: spectral_intensities = X_train xlabel = 'Channel intensity (normalised)' plt.figure() for channel in xrange(spectral_intensities.shape[1]): plt.hist(spectral_intensities[y_train == class_label, channel], bins=100, label=channel_descriptions[channel], zorder=zorders[channel], color=colors[channel]) plt.xlabel(xlabel, fontsize=14) plt.ylabel('number of samples', fontsize=14) plt.legend() plt.show()
def main(argv): data_file = None results_dir = None model_file = None try: opts, args = getopt.getopt( argv, "hd:r:m:", ["data_file=", "results_dir=", "model_file="]) except getopt.GetoptError: print 'incorrect usage' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) for opt, arg in opts: if opt == "-h": print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' elif opt in ("-d", "--data_file"): data_file = arg elif opt in ("-r", "--results__dir"): results_dir = arg elif opt in ("-m", "--model_file"): model_file = arg if data_file is None: print 'data_file was not given' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) try: print 'loading in data...' train_set, validation_set, test_set = load_data_npz(data_file) except IOError: print 'cannot open data_file', data_file # unit scale if using cross entropy cost and not scaled already #print 'unit scaling data' #train_set[0], validation_set[0], test_set[0] = unit_scale([train_set[0],validation_set[0],test_set[0]]) train_set, validation_set, test_set = [ toShared_xy(train_set), toShared_xy(validation_set), toShared_xy(test_set) ] nChannels = 4 corruptionLevels = [0.1, 0.2, 0.3] dropout_rates = [0.0, 0.0, 0.0, 0.0] dimHiddenLayers = [100, 100, 100] nLabels = 4 sgd_opts = { 'epochs_pre': 1, 'min_epochs': 10, 'max_epochs': 10, 'alpha_pre': 0.001, 'alpha_init': 0.1, 'gamma': 0.0001, 'p': 0.75, 'monitor_frequency': 1000 } momentum = 0.9 miniBatchSize = 20 monitoring_to_file = True if results_dir is not None: # train a new model print 'setting up model...' apply_sdA_sgd(train_set, validation_set, test_set, nChannels, dimHiddenLayers, nLabels, miniBatchSize, sgd_opts, dropout_rates, corruptionLevels, results_dir, momentum, monitoring_to_file) elif model_file is not None: # test a saved model # this option will be ignored if results_dir is not None print 'testing saved model...' test_saved_model(test_set, nLabels, model_file) else: print 'exactly 2 arguments must be passed on command line' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>'
def main(argv): data_file = None results_dir = None model_file = None try: opts, args = getopt.getopt(argv, "hd:r:m:", ["data_file=", "results_dir="]) except getopt.GetoptError: print 'incorrect usage' print 'usage: mlp.py -d <data_file> -r <results_dir>' sys.exit(2) for opt, arg in opts: if opt == "-h": print 'usage1: mlp.py -d <data_file> -r <results_dir>' elif opt in ("-d", "--data_file"): data_file = arg elif opt in ("-r", "--results__dir"): results_dir = arg elif opt in ("-m", "--model_file"): model_file = arg if data_file is None: print 'data_file was not given' print 'usage1: mlp.py -d <data_file> -r <results_dir>' print 'usage2: mlp.py -d <data_file> -m <saved_model_file>' sys.exit(2) if results_dir is not None: # train a new model dimHiddenLayers = [100, 100, 100] nLabels = 4 miniBatchSize = 20 sgd_opts = { 'min_epochs': 10, 'max_epochs': 10, 'alpha_init': 0.1, 'gamma': 0.0001, 'p': 0.75, 'monitor_frequency': 1000 } lmbda = 0.0 dropout_rates = [0.0, 0.0, 0.0, 0.5] activations = [relu, relu, relu] momentum = 0.9 monitoring_to_file = True label_descriptions = [ 'rapeseed', 'water', 'built up', 'bare soil', 'wheat', 'grass', 'clouds', 'cloud shadows' ] try: print 'loading in data...' data = load_data_npz(data_file) #plot_bar(np.bincount(train_set[1]), xlabel='landcover class', ylabel='number of samples', label_descriptions=label_descriptions) # landsat 2 remove class that corresponds to border # for i in xrange(len(data)): # keep = data[i][1]!=13 # data[i][0]=data[i][0][keep] # data[i][1]=data[i][1][keep] # data[i][1][data[i][1]==14]=13 # data[i][1][data[i][1]==15]=14 # data[i][1][data[i][1]==16]=15 # print np.unique(data[i][1]) train_set, validation_set, test_set = data except IOError: print 'cannot open data_file', data_file train_set, validation_set, test_set = [ toShared_xy(train_set), toShared_xy(validation_set), toShared_xy(test_set) ] print 'setting up model...' apply_mlp_sgd(train_set, validation_set, test_set, dimHiddenLayers, nLabels, miniBatchSize, sgd_opts, lmbda, dropout_rates, activations, results_dir, momentum, monitoring_to_file) elif model_file is not None: # Test a saved model. # rapideye imageShape = (5000, 5000) zoomParams = { 'zoom': 10, 'x1': 2400, 'x2': 2600, 'y1': 2400, 'y2': 2600 } # origin top left label_descriptions = [ 'rapeseed', 'water', 'built up', 'other', 'wheat', 'grass', 'clouds', 'cloud shadows' ] #landsat2 # imageShape = (8191,8081) # zoomParams = {'zoom':30, 'x1':5100, 'x2':5200, # 'y1':3100, 'y2':3200} # origin top left # label_descriptions = ['rapeseed', 'wheat', 'grass 1', 'grass 2', 'grass 3', 'built up 1', 'built up 2', 'fallow', 'bare soil 1', 'barley', 'built up 3', 'potatoes', 'shallow water', 'bare soil 2', 'clouds', 'cloud shadows'] miniBatchSize = 1000 try: test_set = load_data_npz(data_file, only_test=True) # landsat 2 remove class that corresponds to border # keep = test_set[1]!=13 # test_set[0]=test_set[0][keep] # test_set[1]=test_set[1][keep] # test_set[1][test_set[1]==14]=13 # test_set[1][test_set[1]==15]=14 # test_set[1][test_set[1]==16]=15 # print np.unique(test_set[1]) except IOError: print 'cannot open data_file', data_file X_test, y_test = toShared_xy(test_set) print 'testing saved model...' test_saved_model(X_test, model_file, label_descriptions, imageShape, miniBatchSize, y_test, zoomParams) else: print 'exactly 2 arguments must be passed on command line' print 'usage1: mlp.py -d <data_file> -r <results_dir>' print 'usage2: mlp.py -d <data_file> -m <saved_model_file>'
def main(argv): data_file = None results_dir = None model_file = None try: opts, args = getopt.getopt( argv, "hd:r:m:", ["data_file=", "results_dir=", "model_file="]) except getopt.GetoptError: print 'incorrect usage' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) for opt, arg in opts: if opt=="-h": print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' elif opt in ("-d", "--data_file"): data_file = arg elif opt in ("-r", "--results__dir"): results_dir = arg elif opt in ("-m", "--model_file"): model_file = arg if data_file is None: print 'data_file was not given' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) if results_dir is not None: # Train a new model. nLabels=10 sgd_opts = {'min_epochs':10, 'max_epochs':10, 'alpha_init':0.1, 'gamma':0.0001, 'p':0.75} miniBatchSize = 600 lmbda = 0.0 try: train_set, validation_set, test_set = load_data_pickled(data_file) except IOError: print 'cannot open data_file', data_file train_set, validation_set, test_set = [toShared_xy(train_set), toShared_xy(validation_set), toShared_xy(test_set)] apply_softmax_sgd( train_set, validation_set, test_set, nLabels, miniBatchSize, sgd_opts, lmbda, results_dir) elif model_file is not None: # Test a saved model. nLabels = 4 try: _, _, test_set = load_data_npz(data_file) except IOError: print 'cannot open data_file', data_file test_set = toShared_xy(test_set) test_saved_model(test_set, nLabels, model_file) else: print 'exactly 2 arguments must be passed on command line' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>'
def main(argv): data_file = None results_dir = None model_file = None try: opts, args = getopt.getopt( argv, "hd:r:m:", ["data_file=", "results_dir=", "model_file="]) except getopt.GetoptError: print 'incorrect usage' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) for opt, arg in opts: if opt == "-h": print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' elif opt in ("-d", "--data_file"): data_file = arg elif opt in ("-r", "--results__dir"): results_dir = arg elif opt in ("-m", "--model_file"): model_file = arg if data_file is None: print 'data_file was not given' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) try: print 'loading in data...' train_set, validation_set, test_set = load_data_npz(data_file) except IOError: print 'cannot open data_file', data_file train_set, validation_set, test_set = [ toShared_xy(train_set), toShared_xy(validation_set), toShared_xy(test_set) ] nChannels = 4 dimHiddenLayers = [1000, 1000, 1000] nLabels = 4 sgd_opts = { 'epochs_pre': 15, 'min_epochs': 15, 'max_epochs': 15, 'alpha_pre': 0.001, 'alpha_init': 0.1, 'gamma': 0.0001, 'p': 0.75 } miniBatchSize = 20 persistent_bool = True k = 1 if results_dir is not None: # train a new model print 'setting up model...' apply_dbn_sgd(train_set, validation_set, test_set, nChannels, dimHiddenLayers, nLabels, miniBatchSize, sgd_opts, persistent_bool, k, results_dir) elif model_file is not None: # test a saved model # this option will be ignored if results_dir is not None print 'testing saved model...' test_saved_model(test_set, nLabels, model_file) else: print 'exactly 2 arguments must be passed on command line' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>'
def main(argv): data_file = None results_dir = None model_file = None try: opts, args = getopt.getopt( argv, "hd:r:m:", ["data_file=", "results_dir=", "model_file="]) except getopt.GetoptError: print 'incorrect usage' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) for opt, arg in opts: if opt=="-h": print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' elif opt in ("-d", "--data_file"): data_file = arg elif opt in ("-r", "--results__dir"): results_dir = arg elif opt in ("-m", "--model_file"): model_file = arg if data_file is None: print 'data_file was not given' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>' sys.exit(2) if results_dir is not None: # Train a new model. numFilters = [6, 12] nLabels = 4 dimHiddenSig = 128 filterDim = 5 poolShape = (3,3) sgd_opts = {'min_epochs':50, 'max_epochs':50, 'alpha_init':0.01, 'gamma':0.0001, 'p':0.75, 'monitor_frequency':1000} miniBatchSize = 20 lmbda = 0.0 dropout_rates = [0.0, 0.0, 0.0, 0.0, 0.0] activations = [relu, relu, relu, relu] momentum = 0.9 mc_samples = None monitoring_to_file = True try: print 'loading in data...' train_set, validation_set, test_set = load_data_npz(data_file) except IOError: print 'cannot open data_file', data_file train_set, validation_set, test_set = [toShared_xy(train_set), toShared_xy(validation_set), toShared_xy(test_set)] print 'setting up model...' apply_LeNet_sgd( train_set, validation_set, test_set, numFilters, nLabels, dimHiddenSig, filterDim, poolShape, miniBatchSize, sgd_opts, lmbda, dropout_rates, activations, results_dir, momentum, mc_samples, monitoring_to_file) elif model_file is not None: # Test a saved model. nLabels = 4 plot_shape = (100,100) nSamples = 10000 label_descriptions=['barren', 'trees', 'grassland', 'other'] try: test_set = load_data_npz(data_file, only_test=True) except IOError: print 'cannot open data_file', data_file X_test, y_test = toShared_x(test_set) print 'testing saved model...' test_saved_LeNet(X_test, nLabels, model_file, label_descriptions, plot_shape, mc_dropout=True) else: print 'exactly 2 arguments must be passed on command line' print 'usage1: softmax.py -d <data_file> -r <results_dir>' print 'usage2: softmax.py -d <data_file> -m <saved_model_file>'