コード例 #1
0
ファイル: K_fold.py プロジェクト: SSShawnJ/EMG
def K_fold(K=10,C=0.1,SEGMENT_SIZE=30,thresh=1):
	from numpy import genfromtxt
	
	# path to files
	path_features="Extracted_features/Features_S_"+str(SEGMENT_SIZE)+"_T_"+str(thresh)+".csv"
	path_labels="Extracted_features/Labels_S_"+str(SEGMENT_SIZE)+"_T_"+str(thresh)+".csv"

	# load and shuffle data
	feature,y=load_data(path_features,path_labels)
	feature = np.array(feature)
	y= np.array(y)
	feature,y=shuffle(feature,y)
	

	# start k-fold
	kf = KFold(n_splits=K)
	score=0.0

	iteration=1
	for train, test in kf.split(feature):
		print "iteration:", iteration
		X_train, X_test, y_train, y_test = feature[train], feature[test], y[train], y[test]
		score+= (svm.train(X_train, y_train,  X_test, y_test,c=C))*1.0/K
		iteration+=1

	return score
コード例 #2
0
def flist(request):
    context = {}
    context.update(settings.GLOBAL_SETTINGS)
    fid = request.GET.get('fid', '')
    sid_list = request.GET.get('sid', '')
    vname = request.GET.get('vname', '')
    data = loadData.load_data()
    if sid_list:
        context['select'] = []
        context['select'] = sid_list.split(",")
        context['vname'] = 'unknown virus'
        if vname:
            context['vname'] = vname
    if fid:
        if fid in data:
            context['fid'] = fid
            context['prefix'] = fid[0] + fid[1]
            context['region'] = data[fid]['attr'][0]
            context['district'] = data[fid]['attr'][1]
            context['locality'] = data[fid]['attr'][2]
            context['lat'] = data[fid]['attr'][3]
            context['lng'] = data[fid]['attr'][4]
            context['alt'] = data[fid]['attr'][5]
            context['fsize'] = data[fid]['attr'][6]
            context['img'] = data[fid]['attr'][7]
            context['sample'] = data[fid]['samp']
        else:
            context['ERRMSG'] = 'field ID ' + fid + ' is not correct'
    else:
        context['ERRMSG'] = 'no field was selected'
    return render(request, 'flist.html', context)
コード例 #3
0
def flist(request):
	context = {}
	context.update(settings.GLOBAL_SETTINGS)
	fid = request.GET.get('fid', '')
	sid_list = request.GET.get('sid', '')
	vname = request.GET.get('vname', '')
	data = loadData.load_data()
	if sid_list:
		context['select'] = []
		context['select'] = sid_list.split(",")
		context['vname'] = 'unknown virus'
		if vname:
			context['vname'] = vname
	if fid:
		if fid in data:
			context['fid'] = fid
			context['prefix'] = fid[0] + fid[1]
			context['region'] = data[fid]['attr'][0]
			context['district'] = data[fid]['attr'][1]
			context['locality'] = data[fid]['attr'][2]
			context['lat'] = data[fid]['attr'][3]
			context['lng'] = data[fid]['attr'][4]
			context['alt'] = data[fid]['attr'][5]
			context['fsize'] = data[fid]['attr'][6]
			context['img'] = data[fid]['attr'][7]
			context['sample'] = data[fid]['samp']
		else:
			context['ERRMSG'] = 'field ID ' + fid + ' is not correct'
	else:
		context['ERRMSG'] = 'no field was selected'
	return render(request, 'flist.html', context)
コード例 #4
0
    def run(self):
        x_train, y_train, x_test, y_test = load_data(show=True)

        model = Model(input_shape=x_train[0, :, :, :].shape)
        model.built_model()
        model.summary()
        model.compile()
        model.fit(x_train, y_train, epochs=3)
        model.save('model/mnist')

        self.y_train = y_train
コード例 #5
0
def load_real_samples():
	# load dataset
	files = os.listdir('.')
	if 'train_images.pkl' not in files:
		(trainX, trainy), (_, _) = read_data()
	else:
		(trainX, trainy), (_, _) = load_data()
	# expand to 3d, e.g. add channels
	X = expand_dims(trainX, axis=-1)
	# convert from ints to floats
	X = X.astype('float32')
	# scale from [0,255] to [-1,1]
	X = (X - 127.5) / 127.5
	return [X, trainy]
コード例 #6
0
ファイル: listViews.py プロジェクト: kentnf/spvirome-website
def dlist(request):
	data = loadData.load_data()
	context = {}
	context.update(settings.GLOBAL_SETTINGS)
	fid = request.GET.get('fid', '')
	if fid:
		if fid in data:
			context['fid'] = fid
			context['prefix'] = fid[0] + fid[1]
			context['region'] = data[fid]['attr'][0]
			context['district'] = data[fid]['attr'][1]
			context['locality'] = data[fid]['attr'][2]
			context['lat'] = data[fid]['attr'][3]
			context['lng'] = data[fid]['attr'][4]
			context['alt'] = data[fid]['attr'][5]
			context['fsize'] = data[fid]['attr'][6]
			context['img'] = data[fid]['attr'][7]
			context['sample'] = data[fid]['samp']
		else:
			context['ERRMSG'] = 'field ID ' + fid + ' is not correct'
	else:
		context['ERRMSG'] = 'no field was selected'
	return render(request, 'dlist.html', context)
コード例 #7
0
def analyseResDict(timeStamp, dataSet, dataSetDir, dictDir, save2File, 
                   outImDir):

    #Get relevant data
    train, label, test, test_label, imsize, channels, num_classes = load_data(dataSet, dataSetDir)

    resDictFile = dictDir + "res_dict_" + dataSet  + "_" + timeStamp + ".pkl"
    paramDictFile = dictDir + "param_dict_" + dataSet  +"_" +  timeStamp + ".pkl"
    resDict = uan_utils.load_obj(resDictFile)
    paramDict = uan_utils.load_obj(paramDictFile)
    
    attackFlow = resDict["attackFlow"]
    
    
    #choose a test subset
    subset= range(10)
    testSubSet =test[subset]


    #Reset
    tf.reset_default_graph()
        
    #Target model
    targetModel = make_basic_cnn(nb_classes = paramDict["num_classes"], 
                                      input_shape = (None, paramDict["imSize"][0], 
                                                     paramDict["imSize"][1], 
                                                     paramDict["nc"]))
    #Compare predictions
    origPredsSubset = uan_utils.calcPreds(testSubSet, targetModel, targetModelFile)
    perbPredsSubset = resDict["testPreds"][subset]

    #Visualize results
    visualizePerb(attackFlow, testSubSet, save2File, outImDir, origPredsSubset, perbPredsSubset)
    
    
    return origPredsSubset, perbPredsSubset, paramDict, resDict
コード例 #8
0
        mb_labels[:, :, :, 0:1] = neg_target
        mb_labels[:, :, :, 1:2] = slices_target

        yield slices_input, mb_labels


# Test and save segmentation
def imsave(fname, arr):
    #arr = np.swapaxes(arr, 0, 2)
    sitk_img = sitk.GetImageFromArray(arr)
    sitk.WriteImage(sitk_img, fname)


n_classes = 2
batch_size = 1
train_x, train_y, test_x, test_y = loadData.load_data()

## Training step
FinalModel = build_network(Inputshape=(322, 322, 6), num_class=2)

tbCallback = TensorBoard(log_dir='/input/logs/2DFCCN_6channels_input_run2',
                         histogram_freq=0,
                         write_graph=True,
                         write_images=True)
n_epochs = 500000
FinalModel.fit_generator(iterate_in_mb_train(train_x, train_y),
                         1,
                         epochs=n_epochs,
                         callbacks=[tbCallback],
                         verbose=0,
                         validation_data=iterate_in_mb_test(test_x, test_y),
コード例 #9
0
def train_LeNet(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=90,
               batch_size=20, nkerns=[20, 50], n_hidden=100):
    
    """
    Stochastic gradient descent optimization for a spectral classifier Convolution
    Neural Network

    :type learning_rate: float
    :param learning_rate: learning rate used (for sgd)

    :type L1_reg: float
    :param L1_reg: L1-norm's weight when added to the cost

    :type L2_reg: float
    :param L2_reg: L2-norm's weight when added to the cost (see
    regularization)

    :type n_epochs: int
    :param n_epochs: maximal number of epochs to run the optimizer

    :type dataset: string
    :param dataset: the path of the dataset file

    :type batch_size: int
    :param batch_size: used to compute the number of mini batches for
    training, validation, and testing.

    :type nkerns: list of ints
    :param nkerns: number of kernels on each layer

    :type n_hidden: int
    :param n_hidden: sets the number of perceptrons in the hidden layer
    """
    rng = np.random.RandomState(54302)
                  
    datasets = load_data(rng) ## you will need to write this function

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch

    # start-snippet-1
    x = T.matrix('x')   # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are presented as 1D vector of
                        # [int] labels

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print '... building the model'

    # Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
    # to a 4D tensor, compatible with our LeNetConvPoolLayer
    # (28, 28) is the size of MNIST images.
    layer0_input = x.reshape((batch_size, 1, 204, 1))

    classifier = LeNet(
        rng = rng,
        input = layer0_input,
        batch_size = 20,
        nkerns = nkerns,
        n_in = nkerns[0] * 47,
        n_hidden = 100,
        n_out = 16 # number of classes
    )

    cost = classifier.negative_log_likelihood(y) # add L1, L2 to cost?

    # create a function to compute the mistakes that are made by the model
    test_model = theano.function(
        [index],
        classifier.layer2.errors(y),
        givens={
            x: test_set_x[index * batch_size: (index + 1) * batch_size],
            y: test_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        [index],
        classifier.layer2.errors(y),
        givens={
            x: valid_set_x[index * batch_size: (index + 1) * batch_size],
            y: valid_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )

    params = classifier.params

    # create a list of gradients for all model parameters
    grads = T.grad(cost, params)

    # train_model is a function that updates the model parameters by
    # SGD Since this model has many parameters, it would be tedious to
    # manually create an update rule for each model parameter. We thus
    # create the updates list by automatically looping over all
    # (params[i], grads[i]) pairs.
    updates = [
        (param_i, param_i - learning_rate * grad_i)
        for param_i, grad_i in zip(params, grads)
    ]

    train_model = theano.function(
        [index],
        cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size],
            y: train_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )
    # end-snippet-1

    ###############
    # TRAIN MODEL #
    ###############
    print '... training'
    # early-stopping parameters
    patience = 10000  # look at this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is
                           # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)
                                  # go through this many
                                  # minibatches before checking the network
                                  # on the validation set; in this case we
                                  # check every epoch

    best_validation_loss = np.inf
    best_iter = 0
    test_score = 0.
    start_time = timeit.default_timer()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1 # increment iterator
        for minibatch_index in xrange(n_train_batches): # go through each minibatch

            iter = (epoch - 1) * n_train_batches + minibatch_index

            if iter % 100 == 0:
                print 'training @ iter = ', iter
                  # notify user after each 100
                  # batches
            
            cost_ij = train_model(minibatch_index)
            # calculate cost with this minibatch

            if (iter + 1) % validation_frequency == 0:
                  # If we've covered enough iterations to give validation
                  # a try, do so:
                  
                # compute zero-one loss on validation set
                validation_losses = [validate_model(i) for i
                                     in xrange(n_valid_batches)]
                this_validation_loss = np.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    #join lines with \
                    if this_validation_loss < best_validation_loss *  \
                       improvement_threshold:
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter

                    # test it on the test set
                    test_losses = [
                        test_model(i)
                        for i in xrange(n_test_batches)
                    ]
                    
                    test_score = np.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            # early exit 
            if patience <= iter:
                done_looping = True
                break

    end_time = timeit.default_timer()
    print('Optimization complete.')
    print('Best validation score of %f %% obtained at iteration %i, '
          'with test performance %f %%' %
          (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print >> sys.stderr, ('The code for file ' +
                          os.path.split(__file__)[1] +
                          ' ran for %.2fm' % ((end_time - start_time) / 60.))
コード例 #10
0
 def _read_data(self, security):
     if security in self.security_cache:
         return self.security_cache[security]
     result = loadData.load_data(security)
     self.security_cache[security] = result
     return result
コード例 #11
0
    def __init__(self, args):

        # Parse input arguments
        os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)  # Select GPU device
        self.image_folder = os.path.split(args.dataset.rstrip('/'))[-1]
        batch_size = args.batch
        self.fixedsize = args.fixedsize

        # ======= Data ==========
        print('--- Caching data ---')

        data = load_data(subfolder=self.image_folder)

        self.channels_A = data["nr_of_channels_A"]
        self.img_shape_A = data["image_size_A"] + (self.channels_A, )

        self.channels_B = data["nr_of_channels_B"]
        self.img_shape_B = data["image_size_B"] + (self.channels_B, )

        print('Image A shape: ', self.img_shape_A)
        print('Image B shape: ', self.img_shape_B)

        if self.fixedsize:
            self.input_shape_A = self.img_shape_A
            self.input_shape_B = self.img_shape_B
        else:
            self.input_shape_A = (None, None) + (self.channels_A, )
            self.input_shape_B = (None, None) + (self.channels_B, )
            print('Using unspecified input size')

        self.A_train = data["trainA_images"]
        self.B_train = data["trainB_images"]
        self.A_test = data["testA_images"]
        self.B_test = data["testB_images"]

        self.paired_data = True

        # ===== Model parameters ======
        # Training parameters
        self.lambda_ABA = 10.0  # Cyclic loss weight A_2_B
        self.lambda_BAB = 10.0  # Cyclic loss weight B_2_A
        self.lambda_adversarial = 1.0  # Weight for loss from discriminator guess on synthetic images
        self.learning_rate_D = 2e-4
        self.learning_rate_G = 2e-4
        self.generator_iterations = 1  # Number of generator training iterations in each training loop
        self.discriminator_iterations = 1  # Number of generator training iterations in each training loop
        self.synthetic_pool_size = 50  # Size of image pools used for training the discriminators
        self.beta_1 = 0.5  # Adam parameter
        self.beta_2 = 0.999  # Adam parameter
        self.batch_size = batch_size  # Number of images per batch
        self.epochs = 200  # choose multiples of 20 since the models are saved each 20th epoch

        self.save_models = True  # Save or not the generator and discriminator models
        self.save_models_inteval = 20  # Number of epoch between saves of generator and discriminator models
        self.save_training_img = True  # Save or not example training results or only tmp.png
        self.save_training_img_interval = 1  # Number of epoch between saves of intermediate training results
        self.tmp_img_update_frequency = 3  # Number of batches between updates of tmp.png

        # Architecture parameters
        self.use_instance_normalization = True  # Use instance normalization or batch normalization
        self.use_dropout = False  # Dropout in residual blocks
        self.use_bias = True  # Use bias
        self.use_linear_decay = True  # Linear decay of learning rate, for both discriminators and generators
        self.decay_epoch = 101  # The epoch where the linear decay of the learning rates start
        self.use_patchgan = True  # PatchGAN - if false the discriminator learning rate should be decreased
        self.use_resize_convolution = False  # Resize convolution - instead of transpose convolution in deconvolution layers (uk) - can reduce checkerboard artifacts but the blurring might affect the cycle-consistency
        self.discriminator_sigmoid = True

        # Tweaks
        self.REAL_LABEL = 1.0  # Use e.g. 0.9 to avoid training the discriminators to zero loss

        # ===== Architecture =====
        # Normalization
        if self.use_instance_normalization:
            self.normalization = InstanceNormalization
        else:
            self.normalization = BatchNormalization

        # Optimizers
        self.opt_D = Adam(self.learning_rate_D, self.beta_1, self.beta_2)
        self.opt_G = Adam(self.learning_rate_G, self.beta_1, self.beta_2)

        # Build discriminators
        D_A = self.build_discriminator(self.input_shape_A)
        D_B = self.build_discriminator(self.input_shape_B)

        # Define discriminator models
        image_A = Input(shape=self.input_shape_A)
        image_B = Input(shape=self.input_shape_B)
        guess_A = D_A(image_A)
        guess_B = D_B(image_B)
        self.D_A = Model(inputs=image_A, outputs=guess_A, name='D_A_model')
        self.D_B = Model(inputs=image_B, outputs=guess_B, name='D_B_model')

        # Compile discriminator models
        loss_weights_D = [0.5
                          ]  # 0.5 since we train on real and synthetic images
        self.D_A.compile(optimizer=self.opt_D,
                         loss=self.lse,
                         loss_weights=loss_weights_D)
        self.D_B.compile(optimizer=self.opt_D,
                         loss=self.lse,
                         loss_weights=loss_weights_D)

        # Use containers to make a static copy of discriminators, used when training the generators
        self.D_A_static = Network(inputs=image_A,
                                  outputs=guess_A,
                                  name='D_A_static_model')
        self.D_B_static = Network(inputs=image_B,
                                  outputs=guess_B,
                                  name='D_B_static_model')

        # Do note update discriminator weights during generator training
        self.D_A_static.trainable = False
        self.D_B_static.trainable = False

        # Build generators
        self.G_A2B = self.build_generator(self.input_shape_A,
                                          self.input_shape_B,
                                          name='G_A2B_model')
        self.G_B2A = self.build_generator(self.input_shape_B,
                                          self.input_shape_A,
                                          name='G_B2A_model')

        # Define full CycleGAN model, used for training the generators
        real_A = Input(shape=self.input_shape_A, name='real_A')
        real_B = Input(shape=self.input_shape_B, name='real_B')
        synthetic_B = self.G_A2B(real_A)
        synthetic_A = self.G_B2A(real_B)
        dB_guess_synthetic = self.D_B_static(synthetic_B)
        dA_guess_synthetic = self.D_A_static(synthetic_A)
        reconstructed_A = self.G_B2A(synthetic_B)
        reconstructed_B = self.G_A2B(synthetic_A)

        # Compile full CycleGAN model
        model_outputs = [
            reconstructed_A, reconstructed_B, dB_guess_synthetic,
            dA_guess_synthetic
        ]
        compile_losses = [self.cycle_loss, self.cycle_loss, self.lse, self.lse]
        compile_weights = [
            self.lambda_ABA, self.lambda_BAB, self.lambda_adversarial,
            self.lambda_adversarial
        ]

        self.G_model = Model(inputs=[real_A, real_B],
                             outputs=model_outputs,
                             name='G_model')

        self.G_model.compile(optimizer=self.opt_G,
                             loss=compile_losses,
                             loss_weights=compile_weights)

        # ===== Folders and configuration =====
        self.date_time = time.strftime(
            '%Y%m%d-%H%M%S', time.localtime()) + '-' + self.image_folder

        # Output folder for run data and images
        self.out_dir = os.path.join('runs', self.date_time)
        if not os.path.exists(self.out_dir):
            os.makedirs(self.out_dir)

        if self.save_training_img:
            self.out_dir_images = os.path.join(self.out_dir, 'training_images')
            if not os.path.exists(self.out_dir_images):
                os.makedirs(self.out_dir_images)

        # Output folder for saved models
        if self.save_models:
            self.out_dir_models = os.path.join(self.out_dir, 'models')
            if not os.path.exists(self.out_dir_models):
                os.makedirs(self.out_dir_models)

        self.write_metadata_to_JSON()

        # Don't pre-allocate GPU memory; allocate as-needed
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        K.tensorflow_backend.set_session(tf.Session(config=config))

        # ======= Initialize training ==========
        sys.stdout.flush()
        # plot_model(self.G_A2B, to_file='GA2B_expanded_model_new.png', show_shapes=True)
        self.train(epochs=self.epochs, batch_size=self.batch_size)
コード例 #12
0
import os
import loadData
import numpy as np
import matplotlib.pyplot as plt
from skimage import transform
from skimage.color import rgb2gray

ROOT_PATH = "../../"
train_data_directory = os.path.join(ROOT_PATH, "TrafficSigns/Training")
test_data_directory = os.path.join(ROOT_PATH, "TrafficSigns/Testing")

images, labels = loadData.load_data(train_data_directory)

images28 = [transform.resize(image, (28, 28)) for image in images]

images28 = np.array(images28)

images28 = rgb2gray(images28)

traffic_signs = [300, 2250, 3650, 4000]

for i in range(len(traffic_signs)):
    plt.subplot(1, 4, i + 1)
    plt.axis('off')
    plt.imshow(images28[traffic_signs[i]], cmap="gray")
    plt.subplots_adjust(wspace=0.5)

plt.show()
コード例 #13
0
 def predict(self, imageArray):
     _, self.y_train, _, _ = load_data()
     unique_outputs = np.unique(self.y_train)
     preds = self.model.predict(imageArray)
     prediction = unique_outputs[preds.argmax()]
     return prediction
コード例 #14
0
ファイル: Model.py プロジェクト: madensit/MTDSC-MA
main_path = "dataSet/"

dataset_name = input("Please enter dataset name: \n")

if dataset_name not in ["cats_dogs", "EmergencySound", "Mixed"]:
    print("Please enter a valid dataset name.")
    exit()

model_type = int(
    input("Select your model architecture (1 => DNN, 2 => CNN): \n"))

if model_type not in [1, 2]:
    print("Please enter a valid model architecture number.")
    exit()

data_list, num_labels = load_data(main_path + dataset_name, model_type)

x_train, x_test, y_train, y_test = data_list

filter_size = 2

if model_type == 1:
    model_architecture = "simple"
    model = create_model(num_labels)
else:
    model_architecture = "cnn"
    model = create_model_cnn(num_labels, 40, 500, 1)

    x_train = x_train.reshape(x_train.shape[0], 40, 500, 1)
    x_test = x_test.reshape(x_test.shape[0], 40, 500, 1)
コード例 #15
0
import loadData as loader
from sklearn.neural_network import MLPClassifier
from sklearn import preprocessing
import matplotlib.pyplot as plt

x_train, y_train, x_test, y_test = loader.load_data("../res/readout7.xls")

Y_index = []
x_train = preprocessing.scale(x_train)
x_test = preprocessing.scale(x_test)
d = {}
ha = []

# for Y in range(3, 20):
#     model = MLPClassifier(solver='adam', alpha=1e-5, hidden_layer_sizes=(Y,), random_state=1, max_iter=100000,
#                           momentum=0.9,
#                           activation='logistic', learning_rate_init=0.001, tol=1e-5)
#
#     model.fit(x_train, y_train)
#     preDiff = model.score(x_test, y_test)
#     loss = model.loss_
#     h = {
#         'loss':loss,
#         'pre':preDiff,
#         'y':Y
#     }
#     ha.append(h)
#     d[Y] = model.loss_
# minY = 3
# maxPre = 0
# print(ha)
コード例 #16
0
from keras.optimizers import SGD
from keras.utils import np_utils
from loadData import load_data

batch_size = 32
nb_classes = 10
nb_epoch = 200
data_augmentation = True

# input image dimensions
img_rows, img_cols = 24, 32
# the CIFAR10 images are RGB
img_channels = 3

# the data, shuffled and split between train and test sets
(X_train, Y_train) = load_data()


model = Sequential()

model.add(Convolution2D(32, 3, 3, border_mode='same',
                        input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
コード例 #17
0
def geo_list(request):
    data = loadData.load_data()
    return JsonResponse(data)
コード例 #18
0
ファイル: problem1_b.py プロジェクト: ArminBaz/USC-EE569
def train(dataset):
    # Hyperparameters
    normalize = [True]
    learning_rates = {
        "mnist": [0.0001, 0.0005, 0.001],
        "fashion-mnist": [0.0001, 0.0005, 0.001],
        "cifar": [0.0001, 0.0005, 0.0008]
    }
    weight_decays = [0, 0.0005]
    num_epochs = 200

    # create a textfile to store the accuracies of each run
    f = open(dataset + "_accuracies.txt", "w")

    # dictionary for early stopping of training based on accuracy
    early_stop = {"mnist": 0.99, "fashion-mnist": 0.90, "cifar": 0.65}

    for norm in normalize:
        for learning_rate in learning_rates[dataset]:
            for decay in weight_decays:
                # read in the correct dataset
                train_loader, test_loader = load_data(dataset, norm)

                # define a new model to train
                leNet = LeNet5(dataset)

                # define the loss and optimizer
                loss_fn = nn.CrossEntropyLoss()
                opt = torch.optim.Adam(params=leNet.parameters(),
                                       lr=learning_rate,
                                       weight_decay=decay)

                # initialize the summaryWriter
                writer = SummaryWriter(
                    f'runs/{dataset}/Norm: {norm}, LR: {learning_rate}, Decay: {decay}'
                )

                print(
                    f'Training with Norm: {norm}, LR: {learning_rate}, Decay: {decay}...'
                )
                # Loop through all the epochs
                for epoch in range(num_epochs):
                    # initialize tqdm for a nice progress bar
                    loop = tqdm(enumerate(train_loader),
                                total=len(train_loader),
                                leave=False)
                    # initialize correct to 0
                    correct, total = 0, 0
                    # Loop through the dataloader
                    for _, (X, y) in loop:
                        # Prediction error
                        pred = leNet(X)  # Forward pass
                        loss = loss_fn(pred, y)  # Loss calculation

                        # Backpropagation
                        opt.zero_grad()  # Zero the gradient
                        loss.backward()  # Calculate updates

                        # Gradient Descent
                        opt.step()  # Apply updates

                        # check if correct and update the total number correct
                        correct += (pred.argmax(1) == y).type(
                            torch.float).sum().item()

                        # update the total size with the size of the batch
                        total += len(y)

                        # Update progress bar
                        loop.set_description(f"Epoch [{epoch+1}/{num_epochs}]")
                        loop.set_postfix(loss=loss.item())

                    # calculate the training accuracy
                    train_acc = correct / total

                    # get the testing accuracy
                    test_acc = test(test_loader, leNet, loss_fn)

                    # update the tensorboard summarywriter
                    writer.add_scalar("Training Accuracy", train_acc,
                                      epoch + 1)
                    writer.add_scalar("Testing Accuracy", test_acc, epoch + 1)

                    # check early stopping
                    if test_acc >= early_stop[dataset]:
                        break

                # get the final testing accuracy and output to text file
                final_test_acc = test(test_loader, leNet, loss_fn)
                print(f'Final Test Accuracy: {final_test_acc}')
                f.write(
                    f'Model Params [Norm: {norm}, LR: {learning_rate}, Decay: {decay}] - Final Accuracy after {epoch} epochs : {final_test_acc}'
                )
                f.write('\n\n')
                # close the tensorboard writer
                writer.close()

    f.close()
コード例 #19
0
ファイル: GASEN.py プロジェクト: sizer74/GASEN
# Loading in data
data = pd.read_csv("student-mat.csv", sep=";")
data = data[["G1", "G2", "G3", "studytime", "failures", "absences"]]

# Predict field
predict = "G3"

# Dropping out the predicted field
x = np.array(data.drop([predict], 1))
y = np.array(data[predict])

x_train, x_test, y_train, y_test = sklearn.model_selection.train_test_split(
    x, y, test_size=0.1)

# Retrieve network objects from networks folder
models = load_data()

# Create objective function
objective_function = lambda w: ensemble_fitness(w, models, x_test, y_test,
                                                'mse')

# Set Genetic Algorithm parameters
sol_per_pop = 8
num_parents_mating = 4

# Defining population size
pop_size = (sol_per_pop, len(models))
# Creating the initial population
new_population = np.random.uniform(low=0, high=1, size=pop_size)
print(new_population)
コード例 #20
0
        predictions = predictions.astype(int)

        num_false = (predictions == test_labels).sum()
        lr_acc = 100 * round(num_false / len(predictions), 4)
        regression.append(lr_acc)

    print('Las losowy, średnia: ', statistics.mean(forest), '\n', 'Regresja logistyczna, średnia: ', statistics.mean(regression))


while True:
    print('Wybierz jedną z funkcjonalności:\n1. Porównanie lasu losowego oraz regresji logistycznej\n2.'
          ' Szukanie najlepszych parametrów lasu\n3. Szukanie najlepszych parametrów regresji logistycznej '
          '\n4. Średnia skuteczność z n wykonań \n5. Zakończ')
    choice = input('Wybór:')
    choice = int(choice)

    features, labels = load_data()

    if choice == 1:
        compare(features, labels)
    elif choice == 2:
        forest_search(features, labels)
    elif choice == 3:
        regression_search(features, labels)
    elif choice == 5:
        break
    elif choice == 4:
        multiple(features, labels)
    else:
        continue
コード例 #21
0
ファイル: train.py プロジェクト: dllen/tf-intro
import os
import loadData
import numpy as np
import random
import matplotlib.pyplot as plt

# 教程地址 https://www.jiqizhixin.com/articles/2017-07-30-3
# 数据下载地址
# http://btsd.ethz.ch/shareddata/BelgiumTSC/BelgiumTSC_Training.zip
# http://btsd.ethz.ch/shareddata/BelgiumTSC/BelgiumTSC_Testing.zip
# 加载数据
ROOT_PATH = "../../"
train_data_directory = os.path.join(ROOT_PATH, "TrafficSigns/Training")
test_data_directory = os.path.join(ROOT_PATH, "TrafficSigns/Testing")

images, labels = loadData.load_data(train_data_directory)
test_images, test_labels = loadData.load_data(test_data_directory)

images28 = [transform.resize(image, (28, 28)) for image in images]
images28 = np.array(images28)
images28 = rgb2gray(images28)

test_images28 = [transform.resize(image, (28, 28)) for image in test_images]
test_images28 = rgb2gray(np.array(test_images28))

# 定义变量
x = tf.placeholder(dtype=tf.float32, shape=[None, 28, 28])
y = tf.placeholder(dtype=tf.int32, shape=[None])

# 输入层
layers = tf.contrib.layers
コード例 #22
0
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier

from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix

import pylab as plt

from loadData import load_data
ld = load_data()

import warnings
warnings.filterwarnings('ignore')

target_names = ['Playoff#0', 'Playoff#1']


def run_SVM(X_train, X_test, y_train, y_test):
    # Training the SVM model using X_train and Y_train
    start_time = time.time()
    svm = SVC(kernel='rbf', C=100, gamma=10)
    svm.fit(X_train, y_train)
    print("---Training Time %s seconds ---" % (time.time() - start_time))
    # Classification of X_test using the SVM model
    start_time = time.time()
コード例 #23
0
    



# Get data
dataSet = "MNIST"

if dataSet == "MNIST":
    dataSetDir = ""
    modelfile  = "../savedmodels/basic_cnn_MNIST/basic_cnn.ckpt"
elif dataSet == "CIFAR-10":
    dataSetDir = "../CIFAR-10/cifar-10-batches-py/"
    modelfile  = "../savedmodels/basic_cnn_CIFAR10/basic_cnn_CIFAR10.ckpt"    


X_train, Y_train, X_test, Y_test, imSize, nc, num_classes = load_data(dataSet, dataSetDir)



#Define paramDict
paramDict = {"UANname": "attack",
"advTarget": 0,
"batchSize": 128,
"imFormat": "NHWC",
"imSize": imSize,
"lFlowMax": 100,
"lrate": 0.01 ,
"n_epochs": 100,
"nc": nc,
"seed": 0 ,
"targeted":True ,
コード例 #24
0
def geo_list(request):
	data = loadData.load_data()
	return JsonResponse(data)
コード例 #25
0
	clf.fit(X_train,y_train)
	pred=clf.predict(X_test)
	return accuracy_score(y_test,pred)








if __name__ == '__main__':
	path1="Data/Negative_dataset_noMoves.csv"
	path2="Data/Positive_dataset_typing.csv"

	feature1,y1=load_data(path1)
	feature2,y2=load_data(path2)
	feature1+=feature2
	y1+=y2


	X_train, X_test, y_train, y_test = train_test_split(feature1, y1, test_size=0.3)

	clf = svm.LinearSVC(C=100)
	clf.fit(X_train,y_train)

	pred=clf.predict(X_test)

	print(accuracy_score(y_test,pred))

	
コード例 #26
0
ファイル: routes.py プロジェクト: cyprus247/BannerApp
def load():
    loadData.load_data()
    return render_template("index.html")
コード例 #27
0
filtered_layer = g(x,target)
pred = predict(x)


# Works! I didn't: have the right number of input nodes to the
# hidden layer; and I didn't have target as a single integers. ok!

################################################################################

batch_size = 20
learning_rate = 0.01
nkerns = (20,50)

rng = np.random.RandomState(23455)

datasets = load_data() ## you will need to write this function

train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]

# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

# allocate symbolic variables for the data
index = T.lscalar()  # index to a [mini]batch

# start-snippet-1
x = T.matrix('x')   # the data is presented as rasterized images
コード例 #28
0
from __future__ import division, print_function, absolute_import

import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected,flatten,activation
from tflearn.layers.conv import conv_2d, max_pool_2d,conv_2d_transpose
from tflearn.layers.merge_ops import merge_outputs,merge
from tflearn.layers.estimator import regression
import numpy as np
import loadData

filedir = 'E:/AI/tensorflow/tensorflow/test/edgetest/HED-BSDS/train_pair_test.txt'
src_data_list, label_list = loadData.load_data(filedir)

data = loadData.read_images(src_data_list)  # image data
label = loadData.read_images(label_list)    # label

# Building 'VGG Network'
net_input_data = input_data(shape=[None,224,224,3])

conv1_1 = conv_2d(net_input_data, nb_filter = 64 , filter_size = 3, strides=1, padding = 'same' ,activation='relu')
conv1_2 = conv_2d(conv1_1,        nb_filter = 64 , filter_size = 3, strides=1, padding = 'same' ,activation='relu')
pool1 = max_pool_2d(conv1_2, kernel_size = 2, strides=2)

conv2_1 = conv_2d(pool1,          nb_filter = 128, filter_size = 3, strides=1, padding = 'same' ,activation='relu')
conv2_2 = conv_2d(conv2_1,        nb_filter = 128, filter_size = 3, strides=1, padding = 'same' ,activation='relu')
pool2 = max_pool_2d(conv2_2, kernel_size = 2, strides=2)

conv3_1 = conv_2d(pool2,          nb_filter = 256, filter_size = 3, strides=1, padding = 'same' ,activation='relu')
conv3_2 = conv_2d(conv3_1,        nb_filter = 256, filter_size = 3, strides=1, padding = 'same' ,activation='relu')
conv3_3 = conv_2d(conv3_2,        nb_filter = 256, filter_size = 3, strides=1, padding = 'same' ,activation='relu')
pool3 = max_pool_2d(conv3_3, kernel_size = 2, strides=2)
コード例 #29
0
ファイル: main.py プロジェクト: shmulib/Thesis
        y_mean, y_std = gaussian_process(parameters, scores, n_hidden_choices)
        y_min = min(scores)

        n_hidden = next_parameter_by_ei(y_min, y_mean, y_std, n_hidden_choices)

        if y_min == 0 or n_hidden in parameters:
            # Lowest expected improvement value have been achieved
            break

    min_score_index = np.argmin(scores)
    return parameters[min_score_index]


if __name__ == '__main__':

    x_train, x_test, y_train, y_test, n_classes, n_dimensionality = load_data(
        path_to_file)

    #parses payload
    best_n_hidden = hyperparam_selection(
        train_network,
        n_hidden_range=[50, 1000],
        func_args=[
            x_train, x_test, y_train, y_test, n_classes, n_dimensionality
        ],
        n_iter=6,
    )

    print best_n_hidden
コード例 #30
0
ファイル: train-roberta.py プロジェクト: jszyxw/boolq-project
def main():

    device_ids = [0]

    init_lr = 1e-5
    max_epochs = 10
    max_length = 512
    batch_size = 2
    gradient_accu = 32 // batch_size

    num_label = 2

    train_mode = False

    prev_acc = 0.
    max_acc = 0.

    config = RobertaConfig.from_pretrained('roberta-large-mnli')
    tknzr = RobertaTokenizer.from_pretrained('roberta-large-mnli')

    train_data, test_data = loadData.load_data()
    train_data = train_data + loadData.load_data_aug()

    train_input_ids, train_mask_ids, train_segment_ids, train_label_ids = get_features(
        train_data, max_length, tknzr)
    test_input_ids, test_mask_ids, test_segment_ids, test_label_ids = get_features(
        test_data, max_length, tknzr)

    # print(all_input_ids.shape)

    all_input_ids = torch.cat(train_input_ids, dim=0).long()
    all_input_mask_ids = torch.cat(train_mask_ids, dim=0).long()
    all_segment_ids = torch.cat(train_segment_ids, dim=0).long()
    all_label_ids = torch.Tensor(train_label_ids).long()
    train_dataloader = create_dataloader(all_input_ids,
                                         all_input_mask_ids,
                                         all_segment_ids,
                                         all_label_ids,
                                         batch_size=batch_size,
                                         train=True)

    all_input_ids = torch.cat(test_input_ids, dim=0).long()
    all_input_mask_ids = torch.cat(test_mask_ids, dim=0).long()
    all_segment_ids = torch.cat(test_segment_ids, dim=0).long()
    all_label_ids = torch.Tensor(test_label_ids).long()
    test_dataloader = create_dataloader(all_input_ids,
                                        all_input_mask_ids,
                                        all_segment_ids,
                                        all_label_ids,
                                        batch_size=batch_size,
                                        train=False)

    model = RobertaClassification(config,
                                  num_label=num_label).cuda(device_ids[0])
    model = torch.nn.DataParallel(model, device_ids=device_ids)

    optimizer = transformers.AdamW(model.parameters(), lr=init_lr, eps=1e-8)
    optimizer.zero_grad()
    #scheduler = transformers.get_constant_schedule_with_warmup(optimizer, len(train_dataloader) // (batch_size * gradient_accu))
    #scheduler = transformers.get_linear_schedule_with_warmup(optimizer, len(train_dataloader) // (batch_size * gradient_accu), (len(train_dataloader) * max_epochs * 2) // (batch_size * gradient_accu), last_epoch=-1)

    if not train_mode:
        max_epochs = 1
        model.load_state_dict(torch.load("../model/model-final.ckpt"))

    # foutput = open("answer-roberta-mnli.txt", "w")

    global_step = 0
    for epoch in range(max_epochs):
        model.train()
        if train_mode:
            loss_avg = 0.
            for step, batch in enumerate(
                    tqdm(train_dataloader, desc="Iteration")):
                global_step += 1
                batch = [t.cuda() for t in batch]
                input_id, input_mask, segment_id, label_id = batch
                loss, _ = model(input_id, segment_id, input_mask, label_id)
                loss = torch.sum(loss)
                loss_avg += loss.item()
                loss = loss / (batch_size * gradient_accu)
                loss.backward()
                if global_step % gradient_accu == 0:
                    optimizer.step()
                    optimizer.zero_grad()
                    #if epoch == 0:
                    #scheduler.step()
            print(loss_avg / len(train_dataloader))

        model.eval()

        final_acc = 0.
        num_test_sample = 0
        tot = [0, 0]
        correct = [0, 0]
        for input_id, input_mask, segment_id, label_id in test_dataloader:
            input_id = input_id.cuda()
            input_mask = input_mask.cuda()
            segment_id = segment_id.cuda()
            label_id = label_id.cuda()

            with torch.no_grad():
                loss, logit = model(input_id, segment_id, input_mask, label_id)
            logit = logit.detach().cpu().numpy()
            # print model prediction result
            # print(logit[0], file = foutput)
            # print(logit[1], file = foutput)
            label_id = label_id.to('cpu').numpy()
            acc = np.sum(np.argmax(logit, axis=1) == label_id)
            pred = np.argmax(logit, axis=1)
            for i in range(label_id.shape[0]):
                tot[label_id[i]] += 1
                if pred[i] == label_id[i]:
                    correct[label_id[i]] += 1
            final_acc += acc
            num_test_sample += input_id.size(0)

        print("epoch:", epoch)
        print("final acc:", final_acc / num_test_sample)
        if final_acc / num_test_sample > max_acc:
            max_acc = final_acc / num_test_sample
            print("save...")
            torch.save(model.state_dict(), "model/model.ckpt")
            print("finish")
        print("Max acc:", max_acc)
        '''
        if final_acc / num_test_sample <= prev_acc:
            for param_group in optimizer.param_groups:
                param_group['lr'] = param_group['lr'] * 0.8
        '''
        prev_acc = final_acc / num_test_sample
        tp = correct[1]
        tn = correct[0]
        fp = tot[1] - correct[1]
        fn = tot[0] - correct[0]
        rec = tp / (tp + fn)
        pre = tp / (tp + fp)
        print("recall:{0}, precision:{1}".format(rec, pre))
        print("f:", 2 * pre * rec / (pre + rec))
        print("acc:", (tp + tn) / (tp + tn + fp + fn))
コード例 #31
0
ファイル: vgg.py プロジェクト: wanqx/faceRecognition
    # Variables to update, i.e. trainable variables.
    trainable_variables = conv_net.trainable_variables

    # Compute gradients.
    gradients = g.gradient(loss, trainable_variables)

    # Update W and b following gradients.
    optimizer.apply_gradients(zip(gradients, trainable_variables))


if __name__ == "__main__":
    from loadData import load_data

    # import Data from loadData.py
    (x_train, y_train), (x_test, y_test) = load_data()
    # Convert to float32.
    x_train, x_test = np.array(x_train,
                               np.float32), np.array(x_test, np.float32)
    # Normalize images value from [0, 255] to [0, 1].
    x_train, x_test = x_train / 255., x_test / 255.

    # Use tf.data API to shuffle and batch data.
    train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))
    #  train_data = train_data.repeat().shuffle(1000).batch(batch_size).prefetch(1)
    train_data = train_data.repeat().shuffle(5000).batch(batch_size).prefetch(
        1)
    #  from tensorflow.keras.datasets import mnist
    #  (x_train, y_train), (x_test, y_test) = mnist.load_data()
    #  # Convert to float32.
    #  x_train, x_test = np.array(x_train, np.float32), np.array(x_test, np.float32)