def _get_buckets():
    """ Load the dataset into buckets based on their lengths.
    train_buckets_scale is the inverval that'll help us 
    choose a random bucket later on.
    """
    test_buckets = data.load_data('test_ids.enc', 'test_ids.dec')
    data_buckets = data.load_data('train_ids.enc', 'train_ids.dec')
    train_bucket_sizes = [len(data_buckets[b]) for b in xrange(len(config.BUCKETS))]
    print("Number of samples in each bucket:\n", train_bucket_sizes)
    train_total_size = sum(train_bucket_sizes)
    # list of increasing numbers from 0 to 1 that we'll use to select a bucket.
    train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
                           for i in xrange(len(train_bucket_sizes))]
    print("Bucket scale:\n", train_buckets_scale)
    return test_buckets, data_buckets, train_buckets_scale
def predict():
    """
    An example of how to load a trained model and use it
    to predict labels.
    """

    # loads the saved model
    classifier = pickle.load(open(os.path.join(os.path.split(__file__)[0], 'best_model.pkl')))

    # compile a predictor function
    predict_model = theano.function(
            inputs = [classifier.input],
            outputs = classifier.y_pred)

    # We can test it on some examples from test set
    dataset = 'mnist.pkl.gz'
    datasets = load_data(dataset)
    test_set_x, test_set_y = datasets[2]
    # test_set_x = test_set_x.get_value()

    predicted_values = predict_model(test_set_x[:10])
    print("Predicted values for the first 10 examples in test set:")
    print(predicted_values)

    return 1
Exemple #3
0
def learn():

    
    clf = linear_model.SGDClassifier(penalty="l2",l1_ratio=0,alpha=0.001,class_weight={1:0.3,0:0.7},n_jobs=3)
    
    rd = 100 * 1000

    iter_num = 4
    
    for i in range(iter_num):
        print "round",i
        train = load_data("train",rd)

        train_label = load_label("train")
        train_label = np.array(train_label)

        count = 0
        for ptrain in train:
            print "partial",count
            plabel = train_label[:rd]
            train_label = train_label[rd:]
            if sum(plabel) > 0.2 * len(plabel):
                print "正例个数",sum(plabel)
                assert len(ptrain) == len(plabel)
                clf.partial_fit(ptrain,plabel,classes=[0,1])
            else :
                break
            count += 1
                
        print 100 * "="
            
    print "train_label",len(train_label)

    return clf
Exemple #4
0
def main():
    from argparse import ArgumentParser
    argparser = ArgumentParser()
    argparser.add_argument('team')
    argparser.add_argument('--username', default='asdf7001')
    argparser.add_argument('--password', default='seleniumpython')
    argparser.add_argument('--iterations', type=int, default=1)
    argparser.add_argument('--monitor_url', type=str, default='http://54.149.105.175:9000')
    argparser.add_argument('--challenge', type=str)
    argparser.add_argument('--proxy', action='store_true')
    argparser.add_argument('--browser', type=str, default='phantomjs')
    argparser.add_argument('--data_dir', type=str, default='data/')
    argparser.add_argument('--lib_dir', type=str, default='lib/')
    argparser.add_argument('--kernel_dir', type=str, default='kernel/')
    argparser.add_argument('--kernel', action='store_true')
    argparser.add_argument('--predictor', default='PokeFrequencyPredictor')
    argparser.add_argument("-v", "--verbose", help="increase output verbosity",
                                            action="store_true")
    args = argparser.parse_args()

    with open(args.team) as fp:
        team_text = fp.read()


    pokedata = load_data(args.data_dir)
    showdown = Showdown(
        team_text,
        MonteCarloAgent(20, pokedata),
        args.username,
        pokedata,
        browser=args.browser,
        password=args.password,
    )
    showdown.run(args.iterations, challenge=args.challenge)
Exemple #5
0
def problem2_4_1():
    data, featureNames = load_data()
    kValues = [5, 10, 20]
    for k in kValues:
        iniCenters = orderedCenters(data, k)
        sses = doExperiment(k, data, initialCenters=iniCenters)
        print "SSES for %d: %f" % (k, sses)
def draw_contours():
    print "# Trace contours"

    rows = 5
    cols = 5

    plt.rcParams["figure.figsize"] = (cols * 5, rows * 5)

    fig, axes = plt.subplots(ncols=cols, nrows=rows)
    fig.subplots_adjust(hspace=0, wspace=0)

    for c in range(0, cols):
        krange = arange(0.01, 0.99, 0.01)
        k = float(c) / float(cols) * 0.99 + 0.01

        print "# Load data", k 
        volume = data.load_data(k, smoothness=2) 
        volume = rot90(volume, 1, [0, 1])

        lim = len(volume) / 2
        for r in range(0, rows):
            rp = float(r) / float(rows)
            zind = int(rp * lim)
            z = rp * (ZMAX-ZMIN) + ZMIN
            print "LAYER", r, zind, len(volume), len(axes)
            layer = volume[zind + 1]
            draw_slice(layer, axes[r, c], z, k)

    plt.savefig('contours.png', dpi=100, bbox_inches='tight', pad_inches=0)
Exemple #7
0
def train(limit=-1, net_id=0, train_dir=DEFAULT_TRAINING_DATA, labels_dir=DEFAULT_LABELS, size=256):
    if limit == -1:
        limit_str = "All"
    else:
        limit_str = limit
    print "{}\tLoading {} images in {}".format(now(), limit, train_dir)
    files, labels = data.load_data(train_dir, labels_dir, verbose=0, limit=limit, size=size)

    net = get_net(net_id)
    if not net:
        print '{}\tNo network with id {}'.format(now(), net_id)
        return

    start_timestamp = int(time.time())
    print "{}\tInitialising network...".format(now())
    net.fit(files, labels)
    end_timestamp = int(time.time())
    duration = end_timestamp - start_timestamp
    print "{}\tTraining complete, total duration {} seconds".format(now(), duration)
    file_name = '{}_{}_{}_NET{}'.format(end_timestamp, duration, limit, net_id)

    f = open(file_name, 'w+')

    print "{}\tSaving file to {}".format(now(), file_name)
    cPickle.dump(net, f, protocol=cPickle.HIGHEST_PROTOCOL)

    f.close()
Exemple #8
0
def problem2_4_3():
    data, featureNames = load_data()
    ssesValues = [0] # dummy first value
    for k in range(1, 51):
        sses = doExperiment(k, data)
        ssesValues.append(sses)
        print "SSES for %d: %f" % (k, sses)
    graphSS(ssesValues)
def main():

	
	# Training data consits of 60000 images and 60000 labels
	# Testing data consists of 10000 images and 10000 labels

	# Each image consits of 784 (28x28) pixels each of which contains a value from
	# 0 to 255.0 which corresponds to its darkness or lightness.

	# Each input needs to be a list of numpy arrays to be valid.
	# Load all of the data
	
	print "Loading data..."
	test_images = data.load_data(LIMITED)
	train_images = data.load_data(LIMITED, "train-images.idx3-ubyte", "train-labels.idx1-ubyte")

	print "Normalizing data..."
	X_train, Y_train = data.convert_image_data(train_images)
	X_test, Y_test = data.convert_image_data(test_images)
	X_train = np.array(X_train)
	Y_train = np.array(Y_train)
	X_test = np.array(X_test)
	Y_test = np.array(Y_test)

	if LOAD == False:
		print "Building the model..."
		_model = model.build()
	else:
		print "Loading the model..."
		elements = os.listdir("model")
		if len(elements) == 0:
			print "No models to load."
		else:
			_model = model.load(elements[len(elements)-1])

	if TRAIN == True:
		print "Training the model..."
		model.train(_model, X_train, Y_train, X_test, Y_test)

	if VISUALIZE:
		model.visualize(_model, test_images, VISUALIZE_TO_FILE)

	if TRAIN == True:
		print "Saving the model..."
		model.save(_model)
Exemple #10
0
def main():
    # network_architecture = dict(n_hidden_recog_1=4000, n_hidden_recog_2=1000, n_hidden_gener_1=1000, n_hidden_gener_2=4000, n_input=12562, n_z=500)
    network_architecture = dict(n_hidden_recog_1=4000, # 1st layer encoder neurons
                                n_hidden_recog_2=1000, # 2nd layer encoder neurons
                                n_hidden_gener_1=1000, # 1st layer decoder neurons
                                n_hidden_gener_2=4000, # 2nd layer decoder neurons
                                n_input=12562, n_z=500)
    data = load_data()[0:1000, :]
    vae = train(data, network_architecture, training_epochs=75)
Exemple #11
0
    def loadData():
        sys.path.append('/home/hey/Desktop/MachineLearning-master/DeepLearning Tutorials/dive_into_keras/')
        from data import load_data
        nb_epoch = 5
        batch_size = 100
        nb_class = 10
        data, label = load_data()

        label = np_utils.to_categorical(label, nb_class)
def train(hps):
    """Training loop."""

    model = vgg_model.VGG(hps, FLAGS.mode)
    model.build_graph()

    summary_writer = tf.train.SummaryWriter(FLAGS.train_dir)
    saver = tf.train.Saver(max_to_keep=0)

    sv = tf.train.Supervisor(logdir=FLAGS.log_root,
                             is_chief=True,
                             summary_op=None,
                             save_summaries_secs=60,
                             save_model_secs=600,
                             saver=saver,
                             global_step=model.global_step)
    sess = sv.prepare_or_wait_for_session()

    summary_writer.add_graph(sess.graph)

    step = 0
    lrn_rate = 1e-4

    dataset = load_data().train

    while not sv.should_stop():
        data, labels = dataset.next_batch(32)

        print step,
        (_, summaries, loss, predictions, truth, train_step) = sess.run(
            [model.train_op, model.summaries, model.cost, model.predictions,
             model.labels, model.global_step],
            feed_dict={model.lrn_rate: lrn_rate, model._images:data, model.labels: labels})

        if step < 10000:
            lrn_rate = 1e-4
        elif step < 20000:
            lrn_rate = 1e-6
        elif step < 30000:
            lrn_rate = 1e-8
        else:
            lrn_rate = 1e-10



        step += 1
        if step % 1 == 0:
            summary_writer.add_summary(summaries, train_step)
            print "Step: %d, Loss: %f"%(train_step, loss)
            summary_writer.flush()
            #print predictions.shape, labels.shape

        if step == 100000:
            break
    sv.Stop()
Exemple #13
0
 def predict():
     data,label=load_data()
     index=[i for i in range(len(data))]
     random.shuffle(index)
     data=data[index]
     label=label[index]
     (traindata,testdata)=(data[0:30000],data[0:30000])
     (trainlabel,testlabel)=(label[0:30000],label[0:30000])
     pred_testlabel=model.predict_classes(testdata,batch_size=1,verbose=1)
     num=len(testlabel)
     accuracy=len([1 for i in range(num) if testlabel[i]==pred_testlabel[i]])/float(num)
     print('model accuracy',accuracy)
Exemple #14
0
def funcnn(LR,BS):
    data, label = load_data()
    label = np_utils.to_categorical(label, 10)
    
    model = Sequential()
    
    model.add(Convolution2D(4, 1, 5, 5, border_mode='valid')) 
    model.add(Activation('relu'))
    model.add(Dropout(0.25))
    
    model.add(Convolution2D(8,4, 3, 3, border_mode='valid'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(poolsize=(2, 2)))
    model.add(Dropout(0.25))
    
    model.add(Convolution2D(16, 8, 3, 3, border_mode='valid')) 
    model.add(Activation('relu'))
    model.add(MaxPooling2D(poolsize=(2, 2)))
    model.add(Dropout(0.25))
    
    model.add(Flatten())
    model.add(Dense(16*4*4, 256, init='normal'))
    model.add(Activation('tanh'))
    
    model.add(Dense(256, 10, init='normal'))
    model.add(Activation('softmax'))
    
    sgd = SGD(l2=0.001,lr=LR, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(loss='categorical_crossentropy', optimizer=sgd,class_mode="categorical")
    
    #checkpointer = ModelCheckpoint(filepath="weight.hdf5",verbose=1,save_best_only=True)
    #model.fit(data, label, batch_size=100,nb_epoch=10,shuffle=True,verbose=1,show_accuracy=True,validation_split=0.2,callbacks=[checkpointer])
    result = model.fit(data, label, batch_size=BS,nb_epoch=20,shuffle=True,verbose=1,show_accuracy=True,validation_split=0.2)
    #model.save_weights(weights,accuracy=False)
    
    # plot the result
    
    plt.figure
    plt.plot(result.epoch,result.history['acc'],label="acc")
    plt.plot(result.epoch,result.history['val_acc'],label="val_acc")
    plt.scatter(result.epoch,result.history['acc'],marker='*')
    plt.scatter(result.epoch,result.history['val_acc'])
    plt.legend(loc='under right')
    plt.show()
    
    plt.figure
    plt.plot(result.epoch,result.history['loss'],label="loss")
    plt.plot(result.epoch,result.history['val_loss'],label="val_loss")
    plt.scatter(result.epoch,result.history['loss'],marker='*')
    plt.scatter(result.epoch,result.history['val_loss'],marker='*')
    plt.legend(loc='upper right')
    plt.show()
Exemple #15
0
def main():
    from argparse import ArgumentParser

    argparser = ArgumentParser()
    argparser.add_argument('team1')
    argparser.add_argument('team2')
    argparser.add_argument('--depth', type=int, default=2)
    argparser.add_argument('--gamestate', type=str)
    argparser.add_argument('--player', type=int, default=0)
    args = argparser.parse_args()
    pokedata = load_data("data")

    players = [None, None]
    # players[args.player] = HumanAgent()
    players[args.player] = PessimisticMinimaxAgent(2, pokedata, log_file="normal.txt")
    # players[1 - args.player] = PessimisticMinimaxAgent(2, pokedata, log_file="no_cache.txt", use_cache=False)
    # players[1 - args.player] = DumbAgent()
    players[1 - args.player] = HumanAgent()

    with open(args.team1) as f1, open(args.team2) as f2, open("data/poke2.json") as f3:
        data = json.loads(f3.read())
        poke_dict = Smogon.convert_to_dict(data)
        teams = [Team.make_team(f1.read(), poke_dict), Team.make_team(f2.read(), poke_dict)]

    gamestate = GameState(teams)
    if args.gamestate is not None:
        with open(args.gamestate, 'rb') as fp:
            gamestate = pickle.load(fp)
    with open('cur2.gs', 'wb') as fp:
        pickle.dump(gamestate, fp)
    gamestate.create_gamestate_arff(0)
    gamestate.print_readable_data(0)
    gamestate.print_readable_data(1)
    simulator = Simulator(pokedata)
    while not gamestate.is_over():
        print "=========================================================================================="
        print "Player 1 primary:", gamestate.get_team(0).primary(), gamestate.get_team(0).primary().status
        print "Player 2 primary:", gamestate.get_team(1).primary(), gamestate.get_team(1).primary().status
        print ""

        my_action = players[0].get_action(gamestate, 0)
        opp_action = players[1].get_action(gamestate, 1)
        gamestate = simulator.simulate(gamestate, [my_action, opp_action], 0, log=True)
    if gamestate.get_team(0).alive():
        print "You win!"
        print "Congrats to", gamestate.opp_team
        print "Sucks for", gamestate.my_team
    else:
        print "You lose!"
        print "Congrats to", gamestate.my_team
        print "Sucks for", gamestate.opp_team
    gamestate.turn += 1
Exemple #16
0
def problem2_4_4():
    data, featureNames = load_data()
    k = 6
    iniCenters = orderedCenters(data, k)
    clusters, centers = kmeans(data, k,
                               initialCenters=iniCenters)
    avgAll = avg(data.keys(), data)

    print "average: %s" % avgAll
    print "########################"
    print "cluster # -> (distance from average)"
    print "    [# in cluster]:[items in cluster]"
    print ""
    
    for c in clusters:
        print "cluster %d -> %s: " % (c, distance(avgAll, centers[c]))
        print "    [%d]:%s " % (len(clusters[c]), clusters[c])
        print ""
Exemple #17
0
    def _get_tag_list(self, msg):
        content_type, chat_type, chat_id = telepot.glance(msg)
        if content_type != "photo":
            return None

        file_id = msg["photo"][-1]["file_id"]
        (rating, character, copyright, general) = ("", set(), set(), set())

        db_data = data.load_data(chat_id, file_id)
        if db_data:
            rating    = db_data["rating"]
            character = db_data["character"]
            copyright = db_data["copyright"]
            general   = db_data["general"]

            return (rating, character, copyright, general)

        handler_name = ""
        with tempfile.NamedTemporaryFile() as f:
            self.download_file(file_id, f.name)

            for h in handlers_list:
                res = h.run(f)
                if res:
                    (rating, character, copyright, general) = res
                    handler_name = h.HANDLER_NAME
                    break
                else:
                    # Rewind file for reading with other handler.
                    f.seek(0)

            if not res:
                raise InferenceError("Cannot do inference on this image")

        data.save_data(chat_id, file_id, {
            "rating":    rating,
            "character": character,
            "copyright": copyright,
            "general":   general,
            "time":      msg["date"],
            "handler":   handler_name
        })

        return (rating, character, copyright, general)
Exemple #18
0
def plot(element, dataset = 'mnist.pkl.gz'):
    autoencoder = pickle.load(open(os.path.join(os.path.split(__file__)[0], 'autoencoder.pkl')))
    if element == 'reconstructions':
        print('... plot reconstructions')

        datasets = load_data(dataset)
        test_set_x, test_set_y   = datasets[2]

        rec = theano.function([autoencoder.x], autoencoder.z)
        image = Image.fromarray(tile_raster_images(X = rec(test_set_x[:100]), img_shape = (28, 28), tile_shape= (5, 20), tile_spacing=(1, 1)))
        image.save(os.path.join(os.path.split(__file__)[0], 'autoencoderrec.png'))
    elif element == 'repflds':
        print('... plot receptive fields')
        image = Image.fromarray(tile_raster_images(X=autoencoder.W.get_value(borrow = True).T, img_shape = (28, 28), tile_shape = (5, 20), tile_spacing=(1, 1)))
        image.save(os.path.join(os.path.split(__file__)[0],'autoencoderfilter.png'))
    else:
        print("dot't know how to plot %" % element) 
        print("either use 'reconstructions' or 'repflds'") 
        return -1
Exemple #19
0
def xgb_800_without_mnk():
    data, target = load_data()
    data, target, labels = normalize_data(data, target)
    X_train, X_test, y_train, y_test = train_test_split(data, target)
    reg = xgb.XGBRegressor(max_depth=12, learning_rate=0.007, n_estimators=800)
    reg.fit(X_train, y_train, eval_metric=eval_error, eval_set=[(X_train, y_train), (X_test, y_test)])
    bst = reg.booster()
    fscore = bst.get_fscore()
    submit_model(reg)
    print sorted(fscore.iteritems(), key=lambda b: b[1], reverse=True)
    train_err = reg.evals_result_['validation_0']['error']
    test_err = reg.evals_result_['validation_1']['error']
    ind = np.arange(len(train_err))
    plt.figure()
    plt.plot(ind, train_err, label='train')
    plt.plot(ind, test_err, label='test')
    plt.ylim([0.0, 0.2])
    plt.legend(loc='upper left')
    plt.show()
def train(dataset = 'mnist.pkl.gz'):
    dataset = load_data(dataset)
    data = dataset[0][0].astype('float64')

    start_time = timeit.default_timer()

    results = np.zeros((data.shape[0], 2))
    print('... training barnes-Hut tsne')
    for res, save in zip(bh_tsne(np.copy(data), theta = 0.5), results):
        save[...] = res

    end_time = timeit.default_timer()
    print(('The code for file  ' + os.path.split(__file__)[1] + ' ran for %.2fs' % (end_time - start_time)), file = sys.stderr)

    with open(os.path.join(os.path.split(__file__)[0], 'data.pkl'), 'wb') as f:
        pickle.dump(results, f)

    results = results - np.min(results, axis = 0)
    results = results / np.max(results, axis = 0)
def predict(classifier):
    """
    An example of how to load a trained model and use it
    to predict labels.
    """

    # compile a predictor function
    predict_model = theano.function(
            inputs = [classifier.input],
            outputs = classifier.logRegressionLayer.y_pred)

    # We can test it on some examples from test set
    dataset = 'mnist.pkl.gz'
    datasets = load_data(dataset)
    test_set_x, test_set_y = datasets[2]
    # test_set_x = test_set_x.get_value()

    predicted_values = predict_model(test_set_x[:10])
    print("Predicted values for the first 10 examples in test set:")
    print(predicted_values)

    return 1
Exemple #22
0
def predict_per_cpu_full():
    data, target = load_data()
    data, target, labels = normalize_data(data, target)

    data = data[['C0', 'cpuFull']]
    data['target'] = target
    split_by_types = dict()

    cpu_groups = data.groupby('cpuFull')
    for name, group in cpu_groups:
        X_train, X_test, y_train, y_test = train_test_split(group['C0'].reshape(-1, 1), group['target'])
        split_by_types[str(name)] = {
            'train': {
                'data': X_train,
                'target': y_train
            },
            'test': {
                'data': X_test,
                'target': y_test
            }
        }

    # print split_by_types
    summ = 0.0
    for cpu, data_set in split_by_types.iteritems():
        plt.figure()
        # reg = SGDRegressor(loss='huber', n_iter=100, alpha=0.0)
        reg = RandomForestRegressor(n_estimators=5)
        reg.fit(data_set['train']['data'], data_set['train']['target'])
        test_data = data_set['test']['data']
        y_pred = reg.predict(test_data)
        print mape(data_set['test']['target'], y_pred), cpu
        plt.scatter(test_data, data_set['test']['target'], s=3, color='g', label='actual')
        plt.scatter(test_data, y_pred, s=3, color='r', label='predicted')
        plt.legend(loc='upper left')
        plt.ylabel('mul time')
        plt.title('Category: {}'.format(cpu))
        plt.savefig('imgs/{}.png'.format(cpu))
def plot(dataset = 'mnist.pkl.gz'):
    size = 10000
    print('... plotting the results')
    dataset = load_data(dataset)
    data = dataset[0][0].astype('float64')
    results = pickle.load(open(os.path.join(os.path.split(__file__)[0], 'data.pkl')))

    results = results - np.min(results, axis = 0)
    results = results / np.max(results, axis = 0)

    out = np.zeros((size, size), dtype = 'uint8')
    out[...] = 255
    
    for i in xrange(data.shape[0]):
        xpos = int(results[i][0] * (size - 1000) + 500)
        ypos = int(results[i][1] * (size - 1000) + 500)
        pic = scale_to_unit_interval(data[i].reshape((28, 28)))
        out[xpos:xpos + 28, ypos: ypos + 28] = pic * 255

    print('... saving to file ' + os.path.join(os.path.split(__file__)[0], 'tsne_mnist.png'))

    image = Image.fromarray(out)
    image.save(os.path.join(os.path.split(__file__)[0], 'tsne_mnist.png'))
import os

from data import load_data
from sklearn.decomposition import FactorAnalysis
try:
    import cPickle as pickle
except:
    import pickle

# Factor Analysis

# ================================================================
# Apply factor analysis on the tf-idf matrix and transform raw documents into
# intermediate representation.
docs_tfidf, vocab_tfidf, vocabulary = load_data(subset='all')
n_components = 40
fa = FactorAnalysis(n_components=n_components)
fa.fit(docs_tfidf.toarray())
fa_words = fa.transform(vocab_tfidf.toarray())

# Create a dict to hold the new pca words.
fa_dict = dict(zip(vocabulary, fa_words))

# Store the intermediate representation pca words on disk.
fa_dict_filename = 'fa_dict.pk'
if not os.path.exists(fa_dict_filename):
    fa_dict_file = open(fa_dict_filename, 'w')
    pickle.dump(fa_dict, fa_dict_file)

# Store estimator on dist for further usage.
fa_estimator_filename = 'fa_estimator.pk'
Exemple #25
0
            # print "tf.nn.l2_loss(b2).eval()", tf.nn.l2_loss(b2).eval()
            # print "tf.nn.l2_loss(W3).eval() * 5e-8", tf.nn.l2_loss(W3).eval() * 5e-8
            # print "tf.nn.l2_loss(b3).eval()", tf.nn.l2_loss(b3).eval()
            # regularizers = (tf.nn.l2_loss(W1) * 5e-8 + tf.nn.l2_loss(b1) + tf.nn.l2_loss(W2) * 5e-8 + tf.nn.l2_loss(b2) + tf.nn.l2_loss(W3) * 5e-8 + tf.nn.l2_loss(b3))
            # print "regularizers", regularizers.eval()
            # print "5e-4 * regularizers", 5e-4 * regularizers.eval()

            if (step % 500 == 0):
                print("Minibatch loss at step %d: %f" % (step, l))
                print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
        print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
        print("Time taken:", int(time.time() - start_time))


if __name__ == "__main__":
    _, labels = load_data(True)
    # lda data (x_lda.p) was created using:
    #   data = LDA().fit(data.toarray(), labels.nonzero()[1]).transform(data.toarray())
    # pca data (x_pca.p) was created using:
    #   data = PCA().fit(data_X.toarray()).transform(data_X.toarray())
    data = pickle.load(open("x_lda.p", "rb")).astype(np.float32)

    n = 360  # trials per person
    num_subjects = 9  # num people
    for i in range(num_subjects):
        print "\n=== Using subject", i + 1, "as test set."
        data = np.concatenate((data[n:], data[:n]), axis=0)
        labels = np.concatenate((labels[n:], labels[:n]), axis=0)
        train(data, labels)

        
Exemple #26
0
###################

###################
## Main
###################
force_utf8_hack()
os.system("rm %s" % (DONE_IND_FILE))
os.system("rm %s" % (KILLED_IND_FILE))
os.system("touch %s" % (RUNNING_IND_FILE))

###################
## Read IP to Geolocation Database
###################
if DEBUG2: print "Read IP to Geolocation Database"

located_ips = data.load_data(geo_db_dir + ipinfo_db_filename)
IF_DATA_READ = 1

if DEBUG3: print "  #records=%d" % (len(located_ips))
if DEBUG4:
    for ip in located_ips:
        # print located_ips[ip]
        if 'loc' in located_ips[ip]:
            print "  %s, %s" % (ip, located_ips[ip]['loc'])
        else:
            print "  %s\n    " % (ip)
            print located_ips[ip]

# exit()

###################
Exemple #27
0
if os.path.exists(PARAM_FILE_AUTH):
    auths = list_data.load_data(PARAM_FILE_AUTH)
elif os.path.exists(dns_dir + dns_filename):
    auths = list_data.load_data(dns_dir + dns_filename)
else:
    auths = list_data.load_data(cname_dir + auth_filename)

if DEBUG3: print "  #cnames=%d" % (len(cnames))
if DEBUG3: print "  #auth dns=%d" % (len(auths))

###################
## read done IPs
###################
if DEBUG2: print "Read Done IPs"

ips = data.load_data(ips_dir + ip_dict_filename)
IF_DATA_READ = 1

if DEBUG3: print "  #cnames=%d" % (len(ips))

###################
## DNS Query
###################
if DEBUG2: print "DNS Query"

###################
## DEBUG
# cname = "0.site.51zx.com"
# cnames = ["0.site.51zx.com"]
# auths = ["DNS.RAIDC.COM", "agri-dns02.agri.org.cn", "agri-dns02.agri.org.cn"]
# exit()
Exemple #28
0
    # get cmd line args from user <- config file
    if len(sys.argv) > 1:
        # get configuration
        g = utils.get_config(sys.argv[1])
        # check if we need to process text file
        if g['data_file'] and utils.isEmpty(g['ckpt_path']):
            # check if folder exists, if not create folder
            utils.assert_dir(g['data_path'])
            data.process_data(filename= g['data_file'],
                    path= g['data_path'])
        # check if checkpoint path exists
        utils.assert_dir(g['ckpt_path'])

        try:
            # fetch dataset
            X, Y, idx2ch, ch2idx = data.load_data(path=g['data_path'])
        except:
            print('\n>> Is the folder {} empty?'.format(g['ckpt_path']))
            print('Shouldn\'t it be?')
            sys.exit()

        # training set batch generator
        trainset = utils.rand_batch_gen(X, Y, batch_size= g['batch_size'])

        # build the model
        num_classes = len(idx2ch)
        net = char_rnn.CharRNN(seqlen= X.shape[-1], 
                               num_classes= num_classes,
                               num_layers= g['num_layers'],
                               state_size= g['state_size'],
                               epochs= 100000000,
Exemple #29
0
import glob
from collections import Counter
sys.path.append('..')
from data import load_data

coords = np.load('../../data/subset_conjecture_coords.npy')
counts = [np.sum(np.abs(c)) for c in coords]
plt.figure()
plt.title('Premise Use Histogram')
sns.countplot(counts)
print("zeros: {}".format(sum([1 if x == 0 else 0 for x in counts])))

print()
print()

cts = load_data('../../data/model/train_conjecture_unique_tokens.data')     
counts = list(cts.values())
plt.figure()
plt.title('Conjecture Token Occurrences Histogram')
sns.countplot(counts)

cts = load_data('../../data/model/train_premise_unique_tokens.data')     
counts = list(cts.values())
plt.figure()
plt.title('Premise Token Occurrences Histogram')
sns.countplot(counts)

print()
print()

def iter_premise_subtrees():   
Exemple #30
0
def index():
    data = load_data()
    print(data)
    return render_template('dashboard_view.html', data=data)
import data
import numpy as np
import tensorflow as tf
import time
import tempfile
import zipfile
import os
#import tflite_runtime.interpreter as tflite

(x_train, y_train), (x_test, y_test) = data.load_data()

print("Train data: ", x_train.shape)
print("Train labels: ", y_train.shape)
print("Test data: ", x_test.shape)
print("Test labels: ", y_test.shape)

model_File = "./coralmodels/perforated/"
model_Name = "prunedlite"
model_Path = os.path.join(model_File, model_Name + '.tflite')

interpreter = tf.lite.Interpreter(model_path=model_Path)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

test = x_test.reshape(10000, 1, 32, 32, 3)
output_data = np.zeros([10000, 10])

print(y_test.shape)
print(output_data.shape)
print("start test")
Exemple #32
0
    args.device = device

    logger.setLevel(logging.INFO)
    fmt = logging.Formatter('%(asctime)s: [ %(message)s ]',
                            '%m/%d/%Y %I:%M:%S %p')
    console = logging.StreamHandler()
    console.setFormatter(fmt)
    logger.addHandler(console)
    logfile = logging.FileHandler(args.log_file, 'a')

    logfile.setFormatter(fmt)
    logger.addHandler(logfile)


    if args.test:
        test_graph = load_data(args.test_file)
        model = torch.load(args.load_model)
        model.device = device 

        dev_dataset = TriviaQADataset(test_graph, model, False)
        dev_loader = DataLoader(dataset=dev_dataset,
                              batch_size=1,
                              collate_fn=batcher(device),
                              shuffle=False,
                              num_workers=0)
        model.to(device)
        score, total_list = evaluate(dev_loader, model)
        exit()

    train_graph = load_data(args.train_file)
    dev_graph = load_data(args.dev_file)
Exemple #33
0
from sklearn.neighbors import KNeighborsRegressor
import numpy as np
import pandas as pd
import data


def predict(X_train, y_train, X_test):
    clf = KNeighborsRegressor(n_neighbors=11)
    clf.fit(X_train, y_train)
    result = clf.predict(X_test).reshape(-1, 1)
    return result


if __name__ == "__main__":
    X_train, y_train, X_test, id = data.load_data()
    result = np.expm1(predict(X_train, y_train, X_test))

    ans = np.hstack((id, result))
    ans = pd.DataFrame(ans, columns=['Id', 'SalePrice'])
    ans['Id'] = ans['Id'].astype('Int32')
    ans.to_csv('submission.csv', index=False)
Exemple #34
0
def shoot_mpc_qp(roll_idx):
    print(f'\n=== Model Based Control on Example {roll_idx} ===')
    '''
    load data
    '''
    seq_data = load_data(prepared_names,
                         os.path.join(data_dir,
                                      str(roll_idx) + '.rollout.h5'))
    attrs, states, actions, rel_attrs = [
        to_var(d.copy(), use_gpu=use_gpu) for d in seq_data
    ]

    seq_data = denormalize(seq_data, stat)
    attrs_gt, states_gt, actions_gt = seq_data[:3]
    '''
    setup engine
    '''
    param_file = os.path.join(data_dir,
                              str(roll_idx // args.group_size) + '.param')
    param = torch.load(param_file)
    engine.init(param)
    n_obj = engine.num_obj
    '''
    fit koopman
    '''
    print('===> system identification!')
    fit_data = get_more_trajectories(roll_idx)
    fit_data = [to_var(d, use_gpu=use_gpu) for d in fit_data]
    bs = args.fit_num

    attrs_flat = get_flat(fit_data[0])
    states_flat = get_flat(fit_data[1])
    actions_flat = get_flat(fit_data[2])
    rel_attrs_flat = get_flat(fit_data[3])

    g = model.to_g(attrs_flat, states_flat, rel_attrs_flat, args.pstep)
    g = g.view(torch.Size([bs, args.time_step]) + g.size()[1:])

    G_tilde = g[:, :-1]
    H_tilde = g[:, 1:]
    U_left = fit_data[2][:, :-1]

    G_tilde = get_flat(G_tilde, keep_dim=True)
    H_tilde = get_flat(H_tilde, keep_dim=True)
    U_left = get_flat(U_left, keep_dim=True)

    A, B, fit_err = model.system_identify(G=G_tilde,
                                          H=H_tilde,
                                          U=U_left,
                                          rel_attrs=fit_data[3][:1, 0],
                                          I_factor=args.I_factor)
    '''
    shooting
    '''
    print('===> model based control start!')
    # current can not set engine to a middle state
    assert args.roll_start == 0

    start_step = args.roll_start
    g_start_v = model.to_g(attrs=attrs[start_step:start_step + 1],
                           states=states[start_step:start_step + 1],
                           rel_attrs=rel_attrs[start_step:start_step + 1],
                           pstep=args.pstep)
    g_start = to_np(g_start_v[0])

    if args.env == 'Rope':
        goal_step = args.roll_step + args.roll_start
    elif args.env == 'Soft':
        goal_step = args.roll_step + args.roll_start
    elif args.env == 'Swim':
        goal_step = args.roll_step + args.roll_start

    g_goal_v = model.to_g(attrs=attrs[goal_step:goal_step + 1],
                          states=states[goal_step:goal_step + 1],
                          rel_attrs=rel_attrs[goal_step:goal_step + 1],
                          pstep=args.pstep)
    g_goal = to_np(g_goal_v[0])

    states_start = states_gt[start_step]
    states_goal = states_gt[goal_step]
    states_roll = np.zeros((args.roll_step + 1, n_obj, args.state_dim))
    states_roll[0] = states_start

    control = np.zeros((args.roll_step + 1, n_obj, args.action_dim))
    # control_v = to_var(control, use_gpu, requires_grad=True)
    bar = ProgressBar()
    for step in bar(range(args.roll_step)):
        states_input = normalize([states_roll[step:step + 1]], [stat[1]])[0]
        states_input_v = to_var(states_input, use_gpu=use_gpu)
        g_cur_v = model.to_g(attrs=attrs[:1],
                             states=states_input_v,
                             rel_attrs=rel_attrs[:1],
                             pstep=args.pstep)
        g_cur = to_np(g_cur_v[0])
        '''
        setup parameters
        '''
        T = args.roll_step - step + 1

        A_v, B_v = model.A, model.B
        A_t = to_np(A_v[0]).T
        B_t = to_np(B_v[0]).T

        if not args.baseline:
            Q = np.eye(args.g_dim)
        else:
            Q = np.eye(g_goal.shape[-1])

        if args.env == 'Rope':
            R_factor = 0.01
        elif args.env == 'Soft':
            R_factor = 0.001
        elif args.env == 'Swim':
            R_factor = 0.0001
        else:
            assert False

        R = np.eye(args.action_dim) * R_factor
        '''
        generate action
        '''
        rel_attrs_np = to_np(rel_attrs)[0]
        assert args.optim_type == 'qp'
        if step % args.feedback == 0:
            node_attrs = attrs_gt[0] if args.env in ['Soft', 'Swim'] else None
            u = mpc_qp(g_cur,
                       g_goal,
                       step,
                       T,
                       rel_attrs_np,
                       A_t,
                       B_t,
                       Q,
                       R,
                       node_attrs=node_attrs,
                       actions=to_np(actions[step:]),
                       gt_info=[
                           param, states_gt[goal_step:goal_step + 1],
                           attrs[step:step + T], rel_attrs[step:step + T]
                       ])
        else:
            u = u[1:]
            pass
        '''
        execute action
        '''
        engine.set_action(u[0])  # execute the first action
        control[step] = engine.get_action()
        engine.step()
        states_roll[step + 1] = engine.get_state()
    '''
    render
    '''
    engine.render(states_roll,
                  control,
                  param,
                  act_scale=args.act_scale,
                  video=True,
                  image=True,
                  path=os.path.join(args.shootf,
                                    str(roll_idx) + '.shoot'),
                  states_gt=np.tile(states_gt[goal_step:goal_step + 1],
                                    (args.roll_step + 1, 1, 1)),
                  count_down=True,
                  gt_border=True)

    states_result = states_roll[args.roll_step]

    states_goal_normalized = normalize([states_goal], [stat[1]])[0]
    states_result_normalized = normalize([states_result], [stat[1]])[0]

    return norm(states_goal - states_result), (states_goal, states_result,
                                               states_goal_normalized,
                                               states_result_normalized)
Exemple #35
0
from progressbar import ProgressBar

from config import gen_args
from socket import gethostname

args = gen_args()

os.system("mkdir -p " + args.shootf)

log_path = os.path.join(args.shootf, 'log.txt')
tee = Tee(log_path, 'w')

print_args(args)

print(f"Load stored dataset statistics from {args.stat_path}!")
stat = load_data(args.data_names, args.stat_path)

data_names = ['attrs', 'states', 'actions']
prepared_names = ['attrs', 'states', 'actions', 'rel_attrs']
data_dir = os.path.join(args.dataf, args.shoot_set)

if args.shoot_set == 'extra' and gethostname().startswith('netmit'):
    data_dir = args.dataf + '_' + args.shoot_set
'''
model
'''
# build model
use_gpu = torch.cuda.is_available()
if not args.baseline:
    """ Koopman model"""
    model = CompositionalKoopmanOperators(args,
Exemple #36
0
import tensorflow as tf
import nltk
import numpy as np

# preprocessed data
import data
import data_utils

# load data from pickle and npy files
metadata, idx_q, idx_a = data.load_data(PATH='datasets/cornell_corpus/')
(trainX, trainY), (testX,
                   testY), (validX,
                            validY) = data_utils.split_dataset(idx_q, idx_a)

# parameters
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
batch_size = 32
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024

import seq2seq_wrapper

# In[7]:

model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
                                yseq_len=yseq_len,
                                xvocab_size=xvocab_size,
                                yvocab_size=yvocab_size,
                                ckpt_path='ckpt/cornell_corpus/',
Exemple #37
0
sample_strings = ['Sous le pont Mirabeau coule la Seine.']*4
algo = 'adam'  # adam, sgd

dump_path = os.path.join(os.environ.get('TMP_PATH'), 'handwriting',
                         str(np.random.randint(0, 100000000, 1)[0]))
if not os.path.exists(dump_path):
    os.makedirs(dump_path)

########
# DATA #
########
char_dict, inv_char_dict = cPickle.load(open('char_dict.pkl', 'r'))

# All the data is loaded in memory
train_pt_seq, train_pt_idx, train_str_seq, train_str_idx = \
    load_data('hand_training.hdf5')
train_batch_gen = create_generator(
    True, batch_size,
    train_pt_seq, train_pt_idx, train_str_seq, train_str_idx, chunk=chunk)
valid_pt_seq, valid_pt_idx, valid_str_seq, valid_str_idx = \
    load_data('hand_training.hdf5')
valid_batch_gen = create_generator(
    True, batch_size,
    valid_pt_seq, valid_pt_idx, valid_str_seq, valid_str_idx, chunk=chunk)

##################
# MODEL CREATION #
##################
# shape (seq, element_id, features)
seq_pt = T.tensor3('input', floatX)
seq_str = T.matrix('str_input', 'int32')
Exemple #38
0
        aref = getattr(locale, attr)
        locale.setlocale(aref, '')
        (lang, enc) = locale.getlocale(aref)
        if lang != None:
            try:
                locale.setlocale(aref, (lang, 'UTF-8'))
            except:
                os.environ[attr] = lang + '.UTF-8'


###################
## Main
###################
force_utf8_hack()

cnames = data.load_data(output_dir + cname_dict_filename)
auths = data.load_data(output_dir + auth_dict_filename)
cname_list = set(list_data.load_data(output_dir + cname_filename))
auth_list = set(list_data.load_data(output_dir + auth_filename))
fail_cnames = data.load_data(output_dir + fail_cname_dict_filename)
fail_auths = data.load_data(output_dir + fail_auth_dict_filename)
fail_cname_list = set(list_data.load_data(output_dir + fail_cname_filename))
fail_auth_list = set(list_data.load_data(output_dir + fail_auth_filename))

# data_cname_list = set()
# for hostname in cnames:
#     # print "  " + hostname
#     data_cname_list.update(cnames[hostname])

# print "  # hostnames   = %d" % (len(cnames))
# print "  # cnames      = %d" % (len(data_cname_list))
Exemple #39
0
def train_nn(
        # Hyper-Parameters
        dim_token=100,  # word embeding dimension
        dim_locDiff=10,  # location difference dimension
        dim_cueType=10,  #
        dim_ESP_label=10,
        dim_latent=100,

        lstm_layer_n=50,
        lstm_decoder_layer_n=50,

        n2=50 + 10 + 10,
        ydim0=3,
        ydim1=3,
        # win_size = 2,

        # maxTokens1 = 60, # maximum tokens in sentence 1

        # n_ESP_labels = 3,
        n_cueTypes=4,
        n_vocb_words=4396,  # Vocabulary size
        n_locDiffs=111,  # Location difference size

        end_idx=3194,

        patience=10,  # Number of epoch to wait before early stop if no progress
        max_epochs=100,  # The maximum number of epoch to run
        # dispFreq=10,  # Display to stdout the training progress every N updates
        # decay_c=0.,  # Weight decay for the classifier applied to the U weights.
        lrate=0.01,  # Learning rate for sgd (not used for adadelta and rmsprop)
        dropout_p=1.0,

        optimizer=momentum,
        # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate).

        # maxlen=1000,  # Sequence longer then this get ignored
        batch_size=10,  # The batch size during training.
        inter_cost_margin=0.001,

        # Parameter for extra option
        # noise_std=0.,
        # use_dropout=True,  # if False slightly faster, but worst test error
        # This frequently need a bigger model.
        # reload_model=None,  # Path to a saved model we want to start from.
        # test_size=-1
):
    # Model options
    model_options = locals().copy()
    print('-------------------------------------------------------------')
    print("model options", model_options)
    print('-------------------------------------------------------------')

    # load_data, prepare_data = get_dataset(dataset)

    print('Loading data ... ... ...')
    train, valid, test = data.load_data(path='mydata.pkl', n_words=n_vocb_words)

    print('Building model ... ... ...')

    params_all = init_params(model_options, Wemb_value=data.read_gz_file("word_emb.pkl"))

    # tparams = init_tparams(params)
    tparams_d = init_tparams(params_all[0])
    tparams_g = init_tparams(params_all[1])
    tparams_c = OrderedDict()
    for kk, pp in tparams_d.items():
        tparams_c[kk] = tparams_d[kk]

    for kk, pp in tparams_g.items():
        tparams_c[kk] = tparams_g[kk]

    (x,
     masks,
     x_d_y_fake,
     y,
     x_noises,
     x_maxlens,
     f_D_pred_prob,
     f_D_pred,
     f_G_produce,
     dropouts,
     d_cost,
     g_cost) = Build_Model([tparams_d, tparams_g], model_options)

    d_grads = tensor.grad(d_cost, wrt=list(tparams_d.values()))
    # print(tparams_c)
    g_grads = tensor.grad(g_cost, wrt=list(tparams_c.values()), consider_constant=list(tparams_d.values()),
                          disconnected_inputs='ignore')

    lr = tensor.scalar(name='lr')

    # f_grad_shared, f_update = optimizer(lr, tparams, grads, x, masks, y, cost)

    f_D_grad_shared, f_D_update = optimizer(lr, tparams_d, d_grads,
                                            x + dropouts, masks, x_d_y_fake + y, d_cost)
    # f_G_grad_shared, f_G_update = optimizer(lr, tparams_c, g_grads,
    # x_noise + x_maxlen + x_d_ps + dropouts_g, [], x_g_y_fake + yg, g_cost)
    f_G_grad_shared, f_G_update = optimizer(lr, tparams_c, g_grads,
                                            x + x_noises + x_maxlens, masks, y, g_cost)

    print('training ... ... ...')

    kf_valid = get_minibatches_idx(len(valid[0]), batch_size)
    kf_test = get_minibatches_idx(len(test[0]), batch_size)

    print("%d train examples" % len(train[0]))
    print("%d valid examples" % len(valid[0]))
    print("%d test examples" % len(test[0]))

    # history_errs = []
    best_p = None
    bad_counter = 0
    stop_counter = 0

    # if validFreq == -1:
    # validFreq = len(train[0]) // batch_size
    # if saveFreq == -1:
    # saveFreq = len(train[0]) // batch_size

    # last_training_sum_costs = numpy.inf
    last_ave_of_g_costs = numpy.inf
    last_ave_of_d_costs = numpy.inf

    g_costs_list = []
    d_costs_list = []

    uidx = 0  # the number of update done
    estop = False  # early stop
    # start_time = time.time()
    try:
        for eidx in range(max_epochs):
            n_samples = 0

            # Get new shuffled index for the training set.
            kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True)
            # kf = get_minibatches_idx(99, batch_size, shuffle=True)

            # training_sum_costs = 0

            # ave_of_g_costs_sum = 0
            # ave_of_d_costs_sum = 0

            for train_batch_idx, train_index in kf:
                # uidx += 1
                # use_noise.set_value(1.)

                cur_batch_size = len(train_index)

                # Select the random examples for this minibatch
                x_0 = [train[0][t] for t in train_index]
                x_1 = [train[1][t] for t in train_index]

                x_3 = [train[2][t] for t in train_index]

                y_0 = [train[3][t] for t in train_index]
                y_1 = [train[4][t] for t in train_index]

                y_one_out = [train[5][t] for t in train_index]

                x_0, mask_0, maxlen_0 = data.prepare_data(x_0)
                x_1, mask_1, maxlen_1 = data.prepare_data(x_1)
                x_3, mask_3, maxlen_3 = data.prepare_data(x_3, addIdxNum=2)

                y_0 = numpy.asarray(y_0, dtype='int32')
                y_1 = numpy.asarray(y_1, dtype='int32')
                y_one_out = numpy.asarray(y_one_out, dtype='int32')

                rng = numpy.random.RandomState(9998)
                x0_noise_0 = rng.normal(scale=0.01, size=(cur_batch_size, dim_latent)).astype(config.floatX)
                x1_noise_1 = rng.normal(scale=0.01, size=(cur_batch_size, dim_latent)).astype(config.floatX)
                x3_noise_3 = rng.normal(scale=0.01, size=(cur_batch_size, dim_latent)).astype(config.floatX)

                generated_xs = f_G_produce(x0_noise_0, x1_noise_1, x3_noise_3,
                                           maxlen_0, maxlen_1, maxlen_3,
                                           y_0, y_1)

                # numpy.asarray([3] * cur_batch_size, dtype='int32')#
                generated_x_0 = generated_xs[0]
                generated_x_1 = generated_xs[1]
                generated_x_3 = numpy.concatenate(
                    (  # numpy.random.randint(0, n_cueTypes, (cur_batch_size,)).astype('int32')[None,:],
                        # numpy.random.randint(0, n_locDiffs, (cur_batch_size,)).astype('int32')[None,:],
                        x_3[0:2, :],
                        generated_xs[2]), axis=0)
                generated_m_0 = generated_xs[3]
                generated_m_1 = generated_xs[4]
                generated_m_3 = generated_xs[5]
                generated_y_0 = numpy.random.randint(0, ydim0 - 1, (cur_batch_size,)).astype('int32')
                generated_y_1 = numpy.random.randint(0, ydim1, (cur_batch_size,)).astype('int32')

                x_d_0 = numpy.concatenate((x_0, generated_x_0), axis=1)
                x_d_1 = numpy.concatenate((x_1, generated_x_1), axis=1)

                x_d_3 = numpy.concatenate((x_3, generated_x_3), axis=1)
                y_d_0_fake = numpy.asarray([1] * cur_batch_size + [0] * cur_batch_size, dtype='int32')
                y_d_1_fake = numpy.asarray([1] * cur_batch_size + [0] * cur_batch_size, dtype='int32')
                y_d_3_fake = numpy.asarray([1] * cur_batch_size + [0] * cur_batch_size, dtype='int32')
                # mask_ones_0 = numpy.ones_like(mask_0)
                # mask_ones_1 = numpy.ones_like(mask_1)
                # mask_ones_3 = numpy.ones_like(mask_3)
                mask_d_0 = numpy.concatenate((mask_0, generated_m_0), axis=1)
                mask_d_1 = numpy.concatenate((mask_1, generated_m_1), axis=1)
                mask_d_3 = numpy.concatenate((mask_3, generated_m_3), axis=1)
                y_d_0 = numpy.concatenate((y_0, generated_y_0), axis=0)
                y_d_1 = numpy.concatenate((y_1, generated_y_1), axis=0)

                d_cost = f_D_grad_shared(x_d_0, x_d_1, x_d_3,
                                         dropout_p, 1.0,
                                         mask_d_0, mask_d_1, mask_d_3,
                                         y_d_0_fake, y_d_1_fake, y_d_3_fake,
                                         y_d_0, y_d_1)

                g_cost = f_G_grad_shared(x_0, x_1, x_3,
                                         x0_noise_0, x1_noise_1, x3_noise_3,
                                         16, 16, 12,
                                         mask_0, mask_1, mask_3,
                                         generated_y_0, generated_y_1)

                # print(y_g_0.shape)

                print('\rd_cost = %f   g_cost = %f  @  %d' % (d_cost, g_cost, train_batch_idx), end='')
                # print(cur_batch_size)

                # ave_of_g_costs_sum += g_cost
                # ave_of_d_costs_sum += d_cost

                g_costs_list.append(g_cost)
                d_costs_list.append(d_cost)

                if d_cost < g_cost * 0.8:
                    for i in range(10):
                        f_G_update(0.01)
                        g_cost = f_G_grad_shared(x_0, x_1, x_3,
                                                 x0_noise_0, x1_noise_1, x3_noise_3,
                                                 16, 16, 12,
                                                 mask_0, mask_1, mask_3,
                                                 generated_y_0, generated_y_1)
                        if d_cost / g_cost >= 0.8 and d_cost / g_cost <= 1.0 / 0.8:
                            break
                elif g_cost < d_cost * 0.8:
                    for i in range(10):
                        f_D_update(0.01)
                        d_cost = f_D_grad_shared(x_d_0, x_d_1, x_d_3,
                                                 dropout_p, 1.0,
                                                 mask_d_0, mask_d_1, mask_d_3,
                                                 y_d_0_fake, y_d_1_fake, y_d_3_fake,
                                                 y_d_0, y_d_1)
                        if g_cost / d_cost >= 0.8 and g_cost / d_cost <= 1.0 / 0.8:
                            break
                else:
                    f_D_update(0.01)
                    f_G_update(0.01)

                if train_batch_idx % 100 == 0 or train_batch_idx == len(kf) - 1:
                    print("---Now %d/%d training bacthes @ epoch = %d" % (train_batch_idx, len(kf), eidx))

                if train_batch_idx > 0 and \
                        (train_batch_idx % 500 == 0 or train_batch_idx == len(kf) - 1):

                    cur_ave_of_d_costs = sum(d_costs_list) / len(d_costs_list)
                    cur_ave_of_g_costs = sum(g_costs_list) / len(g_costs_list)

                    print('ave_of_d_costs_sum = %f\tave_of_g_costs_sum = %f' % (cur_ave_of_d_costs, cur_ave_of_g_costs))

                    # print('outputing predicted labels of test set ... ... ...')

                    output_pred_labels(model_options,
                                       f_D_pred, f_D_pred_prob,
                                       data.prepare_data, test, kf_test,
                                       verbose=False, path="test_pred_labels.txt")

                    if cur_ave_of_d_costs >= last_ave_of_d_costs * 0.99 and \
                            cur_ave_of_g_costs >= last_ave_of_g_costs * 0.99:
                        stop_counter += 1

                    last_ave_of_d_costs = cur_ave_of_d_costs
                    last_ave_of_g_costs = cur_ave_of_g_costs

                    print('counter for early stopping : %d/%d' % (stop_counter, patience))

                    del d_costs_list[:]
                    del g_costs_list[:]

                    if stop_counter >= patience:
                        print('Early Stop!')
                        estop = True
                        break

                # end for

            if stop_counter >= patience:
                print('Early Stop!')
                estop = True
                break

            if estop:
                break


    except KeyboardInterrupt:
        print("Training interupted")
def iou(y_true, y_pred):
    def f(y_true, y_pred):
        intersection = (y_true * y_pred).sum()
        union = y_true.sum() + y_pred.sum() - intersection
        x = (intersection + 1e-15) / (union + 1e-15)
        x = x.astype(np.float32)
        return x

    return tf.numpy_function(f, [y_true, y_pred], tf.float32)


if __name__ == "__main__":
    ## Dataset
    path = "CVC-612/"
    (train_x, train_y), (valid_x, valid_y), (test_x, test_y) = load_data(path)

    ## Hyperparameters
    batch = 8
    lr = 1e-4
    epochs = 20

    train_dataset = tf_dataset(train_x, train_y, batch=batch)
    valid_dataset = tf_dataset(valid_x, valid_y, batch=batch)

    model = build_model()

    opt = tf.keras.optimizers.Adam(lr)
    metrics = [
        "acc",
        tf.keras.metrics.Recall(),
Exemple #41
0
def iter_premise_subtrees():   
    for f in glob.glob('../../data/model/train_premise_subtrees_0*'):
        trees = load_data(f)
        print(f)
        for t in trees:
            yield t
Exemple #42
0
from collections import defaultdict

import dash
from dash.dependencies import Output, Input, State
import dash_core_components as dcc
import dash_html_components as html
import plotly.graph_objs as go
import pandas as pd
from flask_caching import Cache

from data import load_data, get_config, get_countires_mapping
from utils import CircleMarkerSizer, options, get_axis


# Loading data
data = load_data()

# Options
DATA_CHOICES = sorted(list(data.keys()))
DEFAULT_X_AXIS = "Life expectancy"
DEFAULT_Y_AXIS = "GDP per capita"
COUNTIRES_MAPPING = get_countires_mapping()
SUPPORTED_COUNTRIES = COUNTIRES_MAPPING.keys()

init_config = get_config(
    data, DEFAULT_X_AXIS, DEFAULT_Y_AXIS, supported_countries=SUPPORTED_COUNTRIES
)

# Creating app
app = dash.Dash(__name__)
app.title = "Poor man's Gapminder"
Exemple #43
0
import numpy as np
from groups_fct import match, mean_notes
from data import load_data

users_id = [10, 24, 36, 50, 15]
notes = [[4,2,3,0,4],[0,0,3,5,4],[1,5,3,1,1],[4,0,3,5,4],[0,5,5,5,0]]
film_notes = [[4,2,3,0,4],[0,0,3,5,4],[1,5,3,1,1],[4,0,3,5,4],[0,5,5,5,0]]
nb_films = 5
nb_users = 5
groups = []
group_notes = []
eps = 4

df = load_data()

for i, value in enumerate(users_id) :
    
    match_result = np.array([])
    
    if (i==0) :
        
        groups.append([i])
        group_notes.append(notes[i])
    
    else :
        
        print ("notes : ", notes[i])
        for j in range (len(groups)):
            ind, result = match(notes[i], group_notes[j])
            match_result = np.append(match_result,result)
            
Exemple #44
0
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
                    level=logging.INFO)

creative_model = keyedvectors.KeyedVectors.load_word2vec_format(
    "checkpoints/creative_model.w2v", binary=True)
ad_model = keyedvectors.KeyedVectors.load_word2vec_format(
    "checkpoints/ad_model.w2v", binary=True)
product_model = keyedvectors.KeyedVectors.load_word2vec_format(
    "checkpoints/product_model.w2v", binary=True)
advertiser_model = keyedvectors.KeyedVectors.load_word2vec_format(
    "checkpoints/advertiser_model.w2v", binary=True)
industry_model = keyedvectors.KeyedVectors.load_word2vec_format(
    "checkpoints/industry_model.w2v", binary=True)

train_ad, train_click, train_user, test_ad, test_click = data.load_data()
# train_record
train_record = pd.merge(train_click, train_ad, on="creative_id")
# test_record
test_record = pd.merge(test_click, test_ad, on="creative_id")

# TODO train embedding
train_grouped = train_record.groupby("user_id")
test_grouped = test_record.groupby("user_id")


def get_embedding_from_grouped(user_id, records, column_name, keep_uid=False):
    if column_name == "ad_id":
        model = ad_model
    elif column_name == "creative_id":
        model = creative_model
Exemple #45
0
def main():
    val_names, data_set = load_data()
    iter_limit = 20

    # Q2.4.1: The values of sum of within group sum of squares for k = 5, k = 10 and k = 20.
    print("Q 2.4.1")
    for k in [5, 10, 20]:
        init_centers = init_centers_first_k(data_set, k)
        centers, clusters, num_iterations = train_kmean(
            data_set, init_centers, iter_limit)
        print("k =",
              str(k) + ": " + str(sum_of_within_group_ss(clusters, centers)))
    print()

    # Q2.4.2: The number of iterations that k-means ran for k = 5.
    print("Q 2.4.2")
    k = 5
    init_centers = init_centers_first_k(data_set, k)
    centers, clusters, num_iterations = train_kmean(data_set, init_centers,
                                                    iter_limit)
    print("k =", str(k) + ", num_iter: " + str(num_iterations))
    print()

    # Q2.4.3: A plot of the sum of within group sum of squares versus k for k = 1 - 50.
    # Please start your centers randomly (choose k points from the dataset at random).
    print("Q 2.4.3")
    SSK = []
    min_ssk = 10000000
    min_ssk_c = 0

    # for k in range(1, 51):
    #     init_centers = init_centers_random(data_set, k)
    #     centers, clusters, num_iterations = train_kmean(data_set, init_centers, iter_limit)
    #     ssk = sum_of_within_group_ss(clusters, centers)
    #     print (str(k) + ", " + str(ssk))
    #     SSK.append(ssk)
    #     if ssk < min_ssk:
    #         min_ssk = ssk
    #         min_ssk_c = k
    # plt.plot([i+1 for i in range(50)], SSK, color='r')
    # plt.show()

    print("Q 2.4.4")
    print("The best k is ", str(min_ssk_c),
          "because it has the minimun ssk = ", str(min_ssk))

    print("Q 2.4.5")
    init_centers = init_centers_random(data_set, 50)
    centers, clusters, num_iterations = train_kmean(data_set, init_centers,
                                                    iter_limit)

    data_set_veg = vect_avg(data_set, 50)
    distances = []
    for i in range(50):
        dis = dist(centers[i], data_set_veg)
        print("the distance between ", str(i + 1),
              "th center and avg of all countries is ", str(dis))
        distances.append(dis)

    plt.plot(distances)
    plt.show()

    plt.plot(distances, color='r')
    plt.ylim((0, 10))
    plt.show()

    print("Q 2.4.6")
    for i in range(len(clusters)):
        for j in range(len(clusters[i])):
            if clusters[i][j]['country'] == 'China':
                print("The country belongs to ", str(i), "the center")
                print("The set contain: ")
                for k in range(len(clusters[i])):
                    if k != j:
                        print(clusters[i][k]['country'])
                sys.exit(0)
import matplotlib.pyplot as plt
from data import train_generator, validation_generator


def augmented_data_distribution():
    batch_size = 1000  #len(train_samples)
    batch_count = 1
    epoch = 40
    non_zero_bias = 1 / (1 + epoch / 5.)
    data = train_generator(train_samples, batch_size, non_zero_bias)

    for i in range(batch_count):
        batch_images, batch_steering = (next(data))
        plt.hist(batch_steering, bins=100)
        plt.show()


def raw_data_distribution():
    batch_size = 1000  #len(train_samples)
    batch_count = 1
    data = validation_generator(train_samples, batch_size=batch_size)

    for i in range(batch_count):
        batch_images, batch_steering = (next(data))
        plt.hist(batch_steering, bins=100)
        plt.show()


train_samples, validation_samples = load_data()
#raw_data_distribution()
augmented_data_distribution()
Exemple #47
0
import resnet
from keras.callbacks import ReduceLROnPlateau, EarlyStopping, CSVLogger, TensorBoard
import numpy as np
import data
from keras.utils import np_utils
from cfg import config

lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
                               cooldown=0,
                               patience=5,
                               min_lr=0.5e-6)
early_stopper = EarlyStopping(min_delta=0.001, patience=10)
csv_logger = CSVLogger(config.csvlogger_path)
logging = TensorBoard(config.logs_path)

x_train, y_train, x_val, y_val = data.load_data(config.train_path,
                                                radio=config.radio)
x_train, y_train, x_val, y_val = data.normalize(x_train, y_train, x_val, y_val)

model = resnet.ResnetBuilder.build_resnet_50(
    (config.img_channels, config.img_rows, config.img_cols), config.nb_classes)
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

if not config.data_augmentation:
    print('Not using data augmentation.')
    model.fit(x_train,
              y_train,
              batch_size=config.batch_size,
              nb_epoch=config.nb_epoch,
              validation_data=(x_val, y_val),
Exemple #48
0
from data import load_data
from classify import classify_fills
import json

fills = [f for f in classify_fills(load_data()) if f.kind != 'native']
for fill in fills:
    print(json.dumps(fill.__dict__))
    Y_test[i] = x[1]
    if i % 1000 == 0:
        print('i: {}/{}'.format(i, len(test_loader)))

Y_training = Y_training.type('torch.LongTensor')
Y_test = Y_test.type('torch.LongTensor')

dat = {'X_train': X_training, 'Y_train': Y_training, 'X_test': X_test, 'Y_test': Y_test, 'nc': nc}
'''

#adsfasdfs

dat = data.load_data(args.dataset,
                     args.dataroot,
                     args.batchSize,
                     device=device,
                     imgsize=args.imageSize,
                     Ntrain=args.Ntrain,
                     Ntest=args.Ntest)

#dat = data.load_data(args.dataset, args.dataroot, args.batchSize,
#                        device=device, imgsize=args.imageSize, Ntrain=args.Ntrain, Ntest=args.Ntest)

#### defining generator
#netG = nets.Generator(args.imageSize, args.nz, args.ngf, dat['nc']).to(device)

#netG = nets.Generator(args.imageSize, args.nz, args.ngf, dat['nc']).to(device)
netG = nets.Generator2(args.imageSize, args.nz, args.ngf, dat['nc']).to(device)

#netG = nets.Generator(args.imageSize, args.nz, args.ngf, dat['nc']).to(device)
netG2 = nets.Generator(args.imageSize, args.nz, args.ngf, dat['nc']).to(device)
Exemple #50
0
                    default=0,
                    help="0:only embedding  1:only feature  2:both")
parser.add_argument('--dropout', type=float, default=0.5)
parser.add_argument('--gpu_id', type=int, default=3)

args = parser.parse_args()
pprint(vars(args))

timer = utils.timer(name='main').tic()
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id)
data_path = args.datadir + args.data
# *******************************
# 使用 warm feature 后结果明显下降
# has_warm_feature = False
# *******************************
data = data.load_data(args)
warm_embeddings = data['warm_embeddings']
cold_features = data['cold_features']
warm_features = data['warm_features']
timer.toc('loaded data')

# build model
classifier = model.Edge_classifier(cold_feature_dim=cold_features.shape[-1],
                                   warm_feature_dim=warm_features.shape[-1],
                                   embed_dim=warm_embeddings.shape[-1],
                                   type=args.type,
                                   lr=args.lr,
                                   n_layers=args.n_layers,
                                   hid_dim=args.hid_dim,
                                   dropout=args.dropout)
classifier.build_model()
else:
  filename = plnode_dir + deploy_node_filename

#######
# filename = plnode_dir + deploy_node_filename
#######

nodes = list_data.load_data(filename)


###################
## get data
###################
if DEBUG2: print "Get Data"

cnames = data.load_data(cname_dir + cname_dict_filename)
auths  = data.load_data(cname_dir + auth_dict_filename)
cname_list = set(list_data.load_data(cname_dir + cname_filename))
auth_list  = set(list_data.load_data(cname_dir + auth_filename))
fail_cnames = data.load_data(cname_dir + fail_cname_dict_filename)
fail_auths  = data.load_data(cname_dir + fail_auth_dict_filename)
fail_cname_list = set(list_data.load_data(cname_dir + fail_cname_filename))
fail_auth_list  = set(list_data.load_data(cname_dir + fail_auth_filename))


###################
## find remain hostnames
###################
if DEBUG2: print "Find Remain Hostnames"

hostnames = set(list_data.load_data(hostname_dir + hostname_filename))
Exemple #52
0
import tensorflow
import data
import network
import matplotlib.pyplot
from tensorflow.keras import backend as K
from tensorflow.keras.models import load_model
from metrics import dice
from tensorflow.keras.callbacks import CSVLogger, ModelCheckpoint
import time

epochs = 10
size_train = 128
batch_size = 4
step_per_epochs = size_train // batch_size

data_train, data_test, label_train, label_test = data.load_data()
train_sequence = data.MouseBrainSequence((data_train, label_train),
                                         size=size_train,
                                         batch_size=batch_size)
data_test, label_test = data.fill_augment(data_test, label_test, batch_size)

model = network.unet_model_3d((128, 128, 128, 1), depth=2)
model.summary()

model_file = "models/unet3d-{}".format(int(time.time()))
checkpoint = ModelCheckpoint("{}.h5".format(model_file), save_best_only=True)
csv_logger = CSVLogger("{}-history.csv".format(model_file),
                       append=False,
                       separator=" ")

model_history = model.fit(
## only valid DNS:
# auths  = list_data.load_data(cname_dir + auth_filename)
auths  = list_data.load_data(dns_dir + dns_filename)
###################################################

if DEBUG3: print "  #cnames=%d" % (len(cnames))
if DEBUG3: print "  #auth dns=%d" % (len(auths))


###################
## read done IPs
###################
if DEBUG2: print "Read Done IPs"

ips = data.load_data(ips_dir + ip_dict_filename)
IF_DATA_READ = 1

if DEBUG3: print "  #cnames=%d" % (len(ips))


###################
## find remain CNAMEs
###################
# if DEBUG2: print "Find Remain CNAMEs"

# done_cnames = ips.keys()
# remain_cnames = list(set(cnames) - set(done_cnames))
# cnames = remain_cnames

Exemple #54
0
def data():
    data = load_data()
    return jsonify(data)
    print("Using the GPU")
    device = torch.device("cuda")
else:
    print("WARNING: You are about to run on cpu, and this will likely run out \
      of memory. \n You can try setting batch_size=1 to reduce memory usage")
    device = torch.device("cpu")

###############################################################################
#
# DATA LOADING & PROCESSING
#
###############################################################################

print('Training on ' + args.train_data_path + '\nTesting on ' +
      args.test_data_path)
train_data = load_data(args.train_data_path)
test_data = load_data(args.test_data_path)

if args.debug:
    print('Nevermind, actually debugging with 10 t/v simple pendulums')
    args.num_train = 100
    args.num_test = 100
    args.seq_len = 5000
    args.vocab_size = 100
    from data import get_simple_pendulums_digitize
    train_data = get_simple_pendulums_digitize(args.num_train, args.seq_len,
                                               args.vocab_size)
    testid_data = get_simple_pendulums_digitize(args.num_test, args.seq_len,
                                                args.vocab_size)

Exemple #56
0
def predict(mean=0.0, std=1.0):
    # load and normalize data
    if mean == 0.0 and std == 1.0:
        imgs_train, _, _ = load_data(train_images_path, num_classes)
        mean = np.mean(imgs_train)
        std = np.std(imgs_train)

    imgs_test, imgs_mask_test, names_test = load_data(test_images_path,
                                                      num_classes)

    mean = np.mean(imgs_test)
    std = np.std(imgs_test)

    original_imgs_test = imgs_test.astype(np.uint8)

    imgs_test -= mean
    imgs_test /= std

    # load model with weights
    #model = unet(num_classes) #Unet model
    #model = get_frontend(imSize,imSize, num_classes) #Dilation model
    model = get_dilation_model_unet(imSize, imSize,
                                    num_classes)  #combination model

    model.load_weights(weights_path)

    # make predictions
    imgs_mask_pred = model.predict(imgs_test, verbose=1)
    # save to mat file for further processing
    if not os.path.exists(predictions_path):
        os.mkdir(predictions_path)

    matdict = {
        'pred': imgs_mask_pred,
        'image': original_imgs_test,
        'mask': imgs_mask_test,
        'name': names_test
    }
    savemat(os.path.join(predictions_path, 'predictions.mat'), matdict)

    # save images with segmentation and ground truth mask overlay
    for i in range(len(imgs_test)):
        pred = imgs_mask_pred[i]
        #print(original_imgs_test.shape)
        image = original_imgs_test[i]
        mask = imgs_mask_test[i]

        # segmentation mask is for the middle slice
        image_rgb = gray2rgb(image[:, :, 0])

        # prediction contour image (add all the predictions)
        pred = (np.round(pred) * 255.0).astype(np.uint8)
        # ground truth contour image (add all the masks)
        mask = (np.round(mask) * 255.0).astype(np.uint8)

        # combine image with contours using red for pred and blue for mask
        pred_rgb = np.array(image_rgb)
        annotation = pred_rgb[:, :, 1]

        #Set all the pixels with the annotation to zero and fill it in with the color
        for c in range(num_classes):
            pred_temp = pred[:, :, c]
            mask_temp = mask[:, :, c]

            pred_temp, contours, _ = cv2.findContours(pred_temp.copy(),
                                                      cv2.RETR_EXTERNAL,
                                                      cv2.CHAIN_APPROX_NONE)
            pred_temp = np.zeros(pred_temp.shape)
            cv2.drawContours(pred_temp, contours, -1, (255, 0, 0), 1)

            mask_temp, contours, _ = cv2.findContours(mask_temp.copy(),
                                                      cv2.RETR_EXTERNAL,
                                                      cv2.CHAIN_APPROX_NONE)
            mask_temp = np.zeros(mask_temp.shape)
            cv2.drawContours(mask_temp, contours, -1, (255, 0, 0), 1)

            pred[:, :, c] = pred_temp
            mask[:, :, c] = mask_temp

            annotation[np.maximum(pred[:, :, c], mask[:, :, c]) == 255] = 0

        pred_rgb[:, :, 0] = pred_rgb[:, :, 1] = pred_rgb[:, :, 2] = annotation

        for c in range(num_classes):
            pred_rgb[:, :, 2] = np.maximum(pred_rgb[:, :, 2], mask[:, :, c])
            pred_rgb[:, :, 1] = np.maximum(
                pred_rgb[:, :, 1], (pred[:, :, c] / 255) * class_colors[c][1])
            pred_rgb[:, :, 2] = np.maximum(
                pred_rgb[:, :, 2], (pred[:, :, c] / 255) * class_colors[c][2])
            pred_rgb[:, :, 0] = np.maximum(
                pred_rgb[:, :, 0], (pred[:, :, c] / 255) * class_colors[c][0])

        imsave(os.path.join(predictions_path, names_test[i] + '.png'),
               pred_rgb)

    return imgs_mask_test, imgs_mask_pred, names_test
###################
## get PlanetLab nodes states
###################
if DEBUG2: print "Get PlanetLab Nodes"

nodes = list_data.load_data(plnode_dir + deploy_node_filename)
if DEBUG3: print "  %d nodes" % (len(nodes))


###################
## get data
###################
if DEBUG2: print "Get Data"

cnames = data.load_data(cname_dir + cname_dict_filename)
auths  = data.load_data(cname_dir + auth_dict_filename)
cname_list = set(list_data.load_data(cname_dir + cname_filename))
auth_list  = set(list_data.load_data(cname_dir + auth_filename))
fail_cnames = data.load_data(cname_dir + fail_cname_dict_filename)
fail_auths  = data.load_data(cname_dir + fail_auth_dict_filename)
fail_cname_list = set(list_data.load_data(cname_dir + fail_cname_filename))
fail_auth_list  = set(list_data.load_data(cname_dir + fail_auth_filename))

IF_HOST_READ = 1


###################
## Check Remote Status -- Prepare Tmp Directory
###################
if DEBUG2: print "Prepare Tmp Directory"
Exemple #58
0
print_steps = 100
train_nums = 30000
buffer_size = 1000
regular_rate = 0.0005

batch_size = 128
channels = 1

resume = True  #是否继续训练模型?
'''
losslist = []
accuracy = []
'''
#os.environ['CUDA_VISIBLE_DEVICES']=gpunum

label_batch, image_batch = data.load_data(tr.train_dir, buffer_size,
                                          batch_size, channels)
regularizer = None  #tf.contrib.layers.l2_regularizer(regular_rate)

logits, tt = tr.process(image_batch,
                        train=True,
                        regularizer=regularizer,
                        channels=channels)

global_step = tf.Variable(0, trainable=False)

prob_batch = tf.nn.softmax(logits)
accuracy_top1_batch = tf.reduce_mean(
    tf.cast(tf.nn.in_top_k(prob_batch, label_batch, 1), tf.float32))
accuracy_top5_batch = tf.reduce_mean(
    tf.cast(tf.nn.in_top_k(prob_batch, label_batch, 5), tf.float32))
accuracy_top10_batch = tf.reduce_mean(
Exemple #59
0
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.advanced_activations import PReLU
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD, Adadelta, Adagrad
from keras.utils import np_utils, generic_utils
from six.moves import range
from data import load_data
import random




#加载数据
data, label = load_data()
#打乱数据
index = [i for i in range(len(data))]
random.shuffle(index)
data = data[index]
label = label[index]
print(data.shape[0], ' samples')

#label为0~9共10个类别,keras要求格式为binary class matrices,转化一下,直接调用keras提供的这个函数
label = np_utils.to_categorical(label, 10)

###############
#开始建立CNN模型
###############

#生成一个model
Exemple #60
0
import matplotlib.cm as cm

#load the saved model
model = cPickle.load(open("model.pkl", "rb"))

#define theano funtion to get output of  FC layer
get_feature = theano.function([model.layers[0].input],
                              model.layers[11].get_output(train=False),
                              allow_input_downcast=False)

#define theano funtion to get output of  first Conv layer
get_featuremap = theano.function([model.layers[0].input],
                                 model.layers[2].get_output(train=False),
                                 allow_input_downcast=False)

data, label = load_data()

# visualize feature  of  Fully Connected layer
#data[0:10] contains 10 images
feature = get_feature(data[0:10])  #visualize these images's FC-layer feature
plt.imshow(feature, cmap=cm.Greys_r)
plt.show()

#visualize feature map of Convolution Layer
num_fmap = 4  #number of feature map
for i in range(num_fmap):
    featuremap = get_featuremap(data[0:10])
    plt.imshow(featuremap[0][i],
               cmap=cm.Greys_r)  #visualize the first image's 4 feature map
    plt.show()