def do_POST(self): # Header self.send_response(200) self.end_headers() # Extract data from request content_len = int(self.headers['Content-Length']) data = self.rfile.read(content_len).decode('utf-8') # Extract dictionary with params data = urllib.parse.parse_qs(data) print('[INFO] Data decoded: ', data) # Extract college name, the param with key 'name' new_name = str(data['name'][0]) # Load database names = load_data() # Check if name already exists in database if new_name in names: self.wfile.write( bytes('[ERR] This name already exists in database', "utf-8")) # If not exists, save new name into database else: names.append(new_name) save_data(names) # Sort and save self.wfile.write( bytes('[INFO] Added {} to database'.format(new_name), "utf-8"))
def main(argv): args = parser.parse_args(argv[1:]) # Fetch the data (train_x, train_y), (test_x, test_y) = data_handler.load_data() #print(train_x) #print (train_y) # Feature columns describe how to use the input. my_feature_columns = [] for key in train_x.keys(): my_feature_columns.append(tf.feature_column.numeric_column(key=key)) # Build 2 hidden layer DNN with 10, 10 units respectively. classifier = tf.estimator.DNNLinearCombinedClassifier( dnn_feature_columns=my_feature_columns, # Two hidden layers of 10 nodes each. dnn_hidden_units=[30, 30, 30, 30], # The model must choose between 2 classes. n_classes=2) #print (my_feature_columns) # Train the Model. classifier.train(input_fn=lambda: data_handler.train_input_fn( train_x, train_y, args.batch_size), steps=args.train_steps) # Evaluate the model. eval_result = classifier.evaluate( input_fn=lambda: data_handler.eval_input_fn(test_x, test_y, args. batch_size)) print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result)) # Generate predictions from the model expected = ['Correct', 'Wrong'] predict_x = { 'TimeOfDay': [90, 3], 'user': [4, 3], 'Operation': [13, 3], 'cli_address': [10, 3], } predictions = classifier.predict( input_fn=lambda: data_handler.eval_input_fn( predict_x, labels=None, batch_size=args.batch_size)) template = ('\nPrediction is "{}" ({:.1f}%), expected "{}"') for pred_dict, expec in zip(predictions, expected): class_id = pred_dict['class_ids'][0] probability = pred_dict['probabilities'][class_id] print( template.format(data_handler.NATURE[class_id], 100 * probability, expec))
def _get_buckets(): """ Load the dataset into buckets based on their lengths. train_buckets_scale is the inverval that'll help us choose a random bucket later on. """ test_buckets = data_handler.load_data('test_ids.enc', 'test_ids.dec') data_buckets = data_handler.load_data('train_ids.enc', 'train_ids.dec') train_bucket_sizes = [ len(data_buckets[b]) for b in range(len(config.BUCKETS)) ] print("Number of samples in each bucket:\n", train_bucket_sizes) train_total_size = sum(train_bucket_sizes) # list of increasing numbers from 0 to 1 that we'll use to select a bucket. train_buckets_scale = [ sum(train_bucket_sizes[:i + 1]) / train_total_size for i in range(len(train_bucket_sizes)) ] print("Bucket scale:\n", train_buckets_scale) return test_buckets, data_buckets, train_buckets_scale
def do_GET(self): # Header self.send_response(200) self.send_header('Content-Type', 'text/html') self.end_headers() # Load from CSV names = load_data() for name in names: # Send each name self.wfile.write(bytes(name, 'utf-8')) if name != names[-1]: self.wfile.write(bytes(',', 'utf-8'))
def evaluate(encoder, k=10, seed=1234, evalcv=False, evaltest=True, use_feats=True): """ Run experiment k: number of CV folds test: whether to evaluate on test set """ traintext, testtext, labels = load_data() trainA = encoder.encode(traintext[0], verbose=False, norm=True) trainB = encoder.encode(traintext[1], verbose=False, norm=True) if evalcv: print 'Running cross-validation...' C = eval_kfold(trainA, trainB, traintext, labels[0], shuffle=True, k=k, seed=seed, use_feats=use_feats) else: C = 4 if evaltest: print 'Computing testing skipthoughts...' testA = encoder.encode(testtext[0], verbose=False, norm=True) testB = encoder.encode(testtext[1], verbose=False, norm=True) if use_feats: train_features = np.c_[np.abs(trainA - trainB), trainA * trainB, feats(traintext[0], traintext[1])] test_features = np.c_[np.abs(testA - testB), testA * testB, feats(testtext[0], testtext[1])] else: train_features = np.c_[np.abs(trainA - trainB), trainA * trainB] test_features = np.c_[np.abs(testA - testB), testA * testB] print 'Evaluating...' clf = LogisticRegression(C=C) clf.fit(train_features, labels[0]) yhat = clf.predict(test_features) acc = clf.score(test_features, labels[1]) f1_score = f1(labels[1], yhat) print 'Test accuracy: ' + str(acc) print 'Test F1: ' + str(f1_score) return acc, f1_score
def trainAE(denoise_ae=True): x_train_target, x_test_target = dh.load_data() if denoise_ae: x_train_noisy, x_test_noisy = denoise(x_train_target, x_test_target) # this is our input placeholder input_dim = x_train_noisy.shape[1] input_img = Input(shape=(input_dim, )) encoded = Dense(encoding_dim, activation='relu')(input_img) encoded = Dense(2500, activation='relu', kernel_regularizer=l2(l2_penalty_ae))(encoded) decoded = Dense(5000, activation='relu', kernel_regularizer=l2(l2_penalty_ae))(encoded) decoded = Dense(input_dim, activation='sigmoid', kernel_regularizer=l2(l2_penalty_ae))(encoded) autoencoder = Model(input_img, decoded) autoencoder.compile(optimizer='adadelta', loss='mse') # autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy') autoencoder.fit(x_train_noisy, x_train_target, epochs=50, batch_size=50, shuffle=True, validation_data=(x_test_noisy, x_test_target), callbacks=[ TensorBoard(log_dir='/tmp/autoencoder'), EarlyStopping(monitor='val_loss', patience=25, mode='auto') ]) autoencoder.save('autoencoder.h5') return autoencoder
def test(**kwargs): opt.parse(kwargs) images, tags, labels = load_data(opt.data_path) y_dim = tags.shape[1] X, Y, L = split_data(images, tags, labels) print('...loading and splitting data finish') img_model = ImgModule(opt.bit) txt_model = TxtModule(y_dim, opt.bit) if opt.load_img_path: img_model.load(opt.load_img_path) if opt.load_txt_path: txt_model.load(opt.load_txt_path) if opt.use_gpu: img_model = img_model.cuda() txt_model = txt_model.cuda() query_L = torch.from_numpy(L['query']) query_x = torch.from_numpy(X['query']) query_y = torch.from_numpy(Y['query']) retrieval_L = torch.from_numpy(L['retrieval']) retrieval_x = torch.from_numpy(X['retrieval']) retrieval_y = torch.from_numpy(Y['retrieval']) qBX = generate_image_code(img_model, query_x, opt.bit) qBY = generate_text_code(txt_model, query_y, opt.bit) rBX = generate_image_code(img_model, retrieval_x, opt.bit) rBY = generate_text_code(txt_model, retrieval_y, opt.bit) if opt.use_gpu: query_L = query_L.cuda() retrieval_L = retrieval_L.cuda() mapi2t = calc_map_k(qBX, rBY, query_L, retrieval_L) mapt2i = calc_map_k(qBY, rBX, query_L, retrieval_L) print('...test MAP: MAP(i->t): %3.3f, MAP(t->i): %3.3f' % (mapi2t, mapt2i))
def evaluate(encoder, k=10, seed=1234, evalcv=False, evaltest=True, norm=False): """ Run experiment k: number of CV folds test: whether to evaluate on test set """ print 'Preparing data...' traintext, testtext = load_data() train, train_labels = prepare_data(traintext) test, test_labels = prepare_data(testtext) train_labels = prepare_labels(train_labels) test_labels = prepare_labels(test_labels) # train, train_labels = shuffle(train, train_labels, random_state=seed) print 'Computing training skipthoughts...' trainF = encoder.encode(train, verbose=False, norm=norm) if evalcv: print 'Running cross-validation...' interval = [2 ** t for t in range(0, 9, 1)] # coarse-grained C = eval_kfold(trainF, train_labels, k=k, scan=interval, seed=seed) else: C = 128 if evaltest: print 'Computing testing skipthoughts...' testF = encoder.encode(test, verbose=False, norm=norm) # scaler = StandardScaler() # trainF = scaler.fit_transform(trainF) # testF = scaler.transform(testF) print 'Evaluating...' clf = LogisticRegression(C=C) clf.fit(trainF, train_labels) acc = clf.score(testF, test_labels) print 'Test accuracy: ' + str(acc) return acc
def do_DELETE(self): self.send_response(200) self.end_headers() # Extract data from request content_len = int(self.headers['Content-Length']) data = self.rfile.read(content_len).decode('utf-8') data = urllib.parse.parse_qs(data) college_name = str(data['name'][0]) # Search for this name into the database # Load database names = load_data() # Check if name already exists in database if college_name in names: names.remove(college_name) self.wfile.write(bytes('[INFO] Deleting name in database', "utf-8")) save_data(names) # Sort and save # If not exists, return an error else: self.wfile.write( bytes( '[ERR] {} was not found in database'.format(college_name), "utf-8"))
def do_PUT(self): self.send_response(200) self.end_headers() # Extract data from request content_len = int(self.headers['Content-Length']) data = self.rfile.read(content_len).decode('utf-8') # Extract dict with parameters data = urllib.parse.parse_qs(data) # Extract params from dict old_name = str(data['name'][0]) new_name = str(data['new_name'][0]) # Search for this name into the database names = load_data() if old_name in names: names.remove(old_name) names.append(new_name) self.wfile.write(bytes('[INFO] Updating name in database', "utf-8")) save_data(names) # Sort and save else: self.wfile.write( bytes('[ERR] {} was not found in database'.format(old_name), "utf-8"))
best_v = tmp_v cur_state = tmp_state[:] cur_w = tmp_w cur_v = tmp_v elif tmp_w < V: if random.uniform(0, 1) < math.exp(delta / t): cur_state = tmp_state[:] cur_w = tmp_w cur_v = tmp_v t *= alpha return best_v if __name__ == '__main__': from data_handler import load_data V, N, items = load_data() start = time() print '======================================' print 'simulated annealing algorithm' t0 = 100 beta = 0.01 alpha = 0.9 max_iters = 10 * N runs = 30 average_v = 0 for i in xrange(runs): best_v = sa(V, N, items, t0, beta, alpha, max_iters) average_v += best_v average_v /= runs
from data_handler import load_data, onehot from layer import layer_rnn import numpy as np import theano from util import load_training_log, plot_confusion_matrix theano.config.floatX = 'float32' theano.config.exception_verbosity = 'high' params = [] # Read MNIST training set, validation set, and test set (X, Y), (Xv, Yv), (Xt, Yt) = load_data('mnist.pkl.gz') Y = onehot(Y) Yv = onehot(Yv) Yt = onehot(Yt) input_dim = X.shape[2] output_dim = Y.shape[1] hidden_dim = 200 mini_batch = 100 num_epochs = 100 lr = np.float32(0.01) n_steps = X.shape[1] # define theano network rnn = layer_rnn(n_steps=n_steps, input_dim=X.shape[2], output_dim=Y.shape[1], hidden_dim=hidden_dim)
"n_samples_test": 5000, "class_labels": [1, 2, 3, 4, 5] } classifier_info = { "nfeatures": 2000, "ngrams": 2, "niterations": 1000, "alpha": 0.1, "lambda": 1 } train_documents, train_labels, val_documents, val_labels, test_documents, test_labels, end_index = data_handler.load_data( data_info["source"], data_info["path"], data_info["n_samples_train"], data_info["n_samples_val"], data_info["n_samples_test"], data_info["class_labels"], is_balanced=data_info["is_balanced"]) print("end_index", end_index) extractor = data_handler.generate_bag_of_ngrams_extractor( train_documents, classifier_info["nfeatures"], classifier_info["ngrams"]) pickle.dump(extractor, open(PATH_TO_EXTRACTOR, "wb")) train_input = data_handler.generate_input(train_documents, extractor) val_input = data_handler.generate_input(val_documents, extractor) train_label_input = np.array(train_labels) val_label_input = np.array(val_labels) print(
from bokeh.models.annotations import Title from bokeh.layouts import gridplot from bokeh.plotting import figure from bokeh.transform import linear_cmap from bokeh.palettes import Spectral6, Category20_19, Reds9 Reds9.reverse() import config import data_handler from svg2 import SVG TAG = 'VASTGUI' data = data_handler.load_data() useful_wordlist = data_handler.get_useful_words() useless_wordlist = data_handler.get_useless_words() keyclusters = data_handler.get_synonym_cluster() heatmap_data = data_handler.load_heatmap_data() full_time_range = list(heatmap_data.time.unique()) mapper = linear_cmap(field_name='right', palette=Spectral6, low=0, high=1) color_bar = ColorBar(color_mapper=mapper['transform'], width=8, location=(0, 0)) prefix_count = OrderedDict() wword_count = OrderedDict()
print '# of hidden\t:\t{}'.format(n_units) print '# of output\t:\t{}'.format(n_label) print 'filter size\t:\t{}x{}x{}'.format(filter_length, filter_width, filter_height) print 'batch size\t:\t{}'.format(batch_size) ###load model model = cntn.CNTN(output_channel, filter_length, filter_width, filter_height, n_units, n_label) cf = L.Classifier(model) optimizer = optimizers.Adam() optimizer.setup(cf) ###load dataset training = open(train_url).readlines() dataset = dh.load_data(training, doc_len, word_len) x_train = dataset['source'] keyword_train = dataset['keyword'] y_train = dataset['target'] docs = dataset['docs'] words = dataset['words'] N = len(training) print '# of training\t:\t{}'.format(N) ###gpu setting if gpu == -1: xp = np print '###\tUsing CPU' else:
sliding_window_step = 12 # timesteps = 24 rnn_size = 258 max_len = 150 learn_rate = 0.001 os.environ['CUDA_VISIBLE_DEVICES'] = str(0) seed = 1 exist = os.path.isfile('./data/oppChallenge_gestures.data') if not exist: # Preprocess original OPPORTUNITY zip file preprocess_data.generate_data("./data/OpportunityUCIDataset.zip", "oppChallenge_gestures.data", 'gestures') # Load dataset inputs, labels, test_inputs, test_labels = data.load_data( "./data/oppChallenge_gestures.data") # Divide data into sliding windows of length "sliding_window_length" with steps of "sliding_window_step" inputs, labels = data.opp_sliding_window(inputs, labels, sliding_window_length, sliding_window_step) test_inputs, test_labels = data.opp_sliding_window(test_inputs, test_labels, sliding_window_length, sliding_window_step) # Create Placeholders of shape (n_x, n_y) X = tf.placeholder(tf.float32, [None, sliding_window_length, 1, num_features], name="X") Y = tf.placeholder(tf.float32, [None, n_classes], name="Y")
doc_len = arg.dlen word_len = arg.wlen word_dim = arg.wdim n_units = arg.hdim n_label = arg.label filter_length = arg.flen filter_width = word_len filter_height = word_dim output_channel = arg.c batch_size = arg.b n_epoch = arg.e model_url = arg.model ###load dataset testing = open(testing_url).readlines() dataset = dh.load_data(testing, doc_len, word_len) x_train = dataset['source'] keyword_train = dataset['keyword'] y_train = dataset['target'] docs = dataset['docs'] words = dataset['words'] ###load model print 'model:\t\t{}'.format(model_url) print 'predicted files:{}'.format(testing_url) model = cntn.CNTN(output_channel, filter_length, filter_width, filter_height, n_units, n_label) cf = L.Classifier(model) optimizer = optimizers.Adam()
import tensorflow as tf import data_handler import numpy as np tf.logging.set_verbosity(tf.logging.DEBUG) sess = tf.Session() # Import CIFAR-10 data using data_helpers.py to unpack data_sets = data_handler.load_data() # restore the saved model ( Import the data from the Training session) new_saver = tf.train.import_meta_graph('data/mnist_model/00000001/export.meta') new_saver.restore(sess, 'data/mnist_model/00000001/export') # print to see the restored variables for v in tf.get_collection('variables'): print(v.name) print(sess.run(tf.global_variables())) # Initialize weight and bias ; get saved weights ( using existing weights and bias from the trained model ) W = tf.get_collection('variables')[0] b = tf.get_collection('variables')[1] # placeholders for test images and labels labels_placeholder = tf.placeholder(tf.int64, shape=[None]) images_placeholder = tf.placeholder(tf.float32, shape=[None, 3072]) # predict equation y = tf.nn.softmax(tf.matmul(images_placeholder, W) + b, name='y') # compare predicted label and actual label
while True: data = conn.recv(2048) data = data.decode('utf-8') if data: response = data_handler.get_database(data, files) msg = json.dumps(response).encode() # try here conn.sendall(msg) # Server Setup tcpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tcpServer.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) tcpServer.bind((socket.gethostname(), 1234)) threads = [] # Data Loading files = data_handler.load_data() # Listening tcpServer.listen(5) while True: tcpServer.listen(5) conn, (the_ip, the_port) = tcpServer.accept() new_thread = ClientThread(the_ip, the_port) new_thread.start() threads.append(new_thread)
from data_handler import load_data ReLU = np.vectorize(lambda x: max(0.0, x)) identity = np.vectorize(lambda x: x) shape = [28 * 28, 1200, 1200, 10] activations = [ReLU, ReLU, identity] layers = [] net = "hinton_backprop" epoch = 1200 dropout = False num_samples = 200 for class_label in range(1): datasets = load_data("data/mnist.pkl.gz", [class_label]) train_set = datasets[0] train_x = train_set[0][0:num_samples, :] train_y = train_set[1][0:num_samples] sum_values = np.array([np.zeros(shape[1]), np.zeros(shape[2]), np.zeros(shape[3])]) print "!!! Current class label: {} !!!".format(class_label) for epoch in [2700]: print "Current epoch: ", epoch for i in range(len(shape) - 1): fName = "snapshots/{0}/params_{1}_layer_{2}.txt".format(net, epoch, i) with open(fName, "rb") as f: read_data = f.read() rows = read_data.split(";\n") rows = rows[:-1]
# [534,175,578,204,1]]) # # label2=np.array([1,2,3,4]) # # # print(label_gt) # # image_dim=[[ 375, 1242]] # print(label_gt,image_dim) rpn = [] feat_stride = 16 anchor_scale = [8, 16, 32] data_handler = data_handler.DataHandler() data = "../../Datasets/kitti/train.txt" label_file = "../../Datasets/kitti/label.txt" data_handler.get_file_list(data, label_file) data1, labels1, ims = data_handler.load_data(0, 1) print(data1.shape, labels1.shape, ims.shape) print(labels1, ims) # rpn = region_proposal_network.RegionProposalNetwork(feature_vector=data1,ground_truth=labels1,im_dims=ims,anchor_scale=anchor_scale,Mode="train") rpn_label, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = anchor_target_layer.anchor_target_layer_python( rpn_cls_score=data1, gt_boxes=labels1, im_dims=ims, feat_strides=feat_stride, anchor_scales=anchor_scale) (rpn_label) # rpn_softmax.rpn_soft
str(mating_rate) + ',' + str(variation_rate) + ',' + str(N) + ',' + str(distance) + ',' + str(end - start) + '\n') #bar.update() f.close() return res if __name__ == '__main__': #群体规模N N = 25 #交配概率 mating_rate = 0.8 #变异概率 variation_rate = 0.1 #加载数据 data = data_handler.load_data('data/cn144_location2.txt') ts = TravelingSalesman(data, N, mating_rate, variation_rate) ts.travel_start(200000) print('result:') sequence = ts.ga.best_life.chromosome print(sequence) distance = utils.get_distance_all(ts.ga.best_life.chromosome, data) print(distance) data_handler.draw_path(sequence, data) # res = parameter_selection(data) # res = np.array(res) # print(res)
from deepConvLSTM import deepConvLSTM import data_handler as data import os from keras.optimizers import Adam if __name__ == '__main__': num_epochs = 1000 n_classes = 18 batch_size = 85 num_features = 113 timesteps = 24 rnn_size = 258 max_len = 150 learn_rate = 0.001 os.environ['CUDA_VISIBLE_DEVICES'] = str(0) inputs, labels, test_inputs, test_labels = data.load_data() shape = inputs.shape[1:] model = deepConvLSTM(n_classes, shape) opt = Adam(lr=learn_rate) model.compile(opt, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(inputs, labels, batch_size=batch_size, epochs=num_epochs, validation_split=0.2) loss, acc = model.evaluate(test_inputs, test_labels, steps=1000) print("Model accuraccy:", acc, ", loss:", loss)
def train(**kwargs): opt.parse(kwargs) images, tags, labels = load_data(opt.data_path) y_dim = tags.shape[1] X, Y, L = split_data(images, tags, labels) print('...loading and splitting data finish') img_model = ImgModule(opt.bit) txt_model = TxtModule(y_dim, opt.bit) if opt.use_gpu: img_model = img_model.cuda() txt_model = txt_model.cuda() train_L = torch.from_numpy(L['train']).float() train_x = torch.from_numpy(X['train']).float() train_y = torch.from_numpy(Y['train']).float() query_L = torch.from_numpy(L['query']).float() query_x = torch.from_numpy(X['query']).float() query_y = torch.from_numpy(Y['query']).float() retrieval_L = torch.from_numpy(L['retrieval']).float() retrieval_x = torch.from_numpy(X['retrieval']).float() retrieval_y = torch.from_numpy(Y['retrieval']).float() # gc del images, tags, labels, L, X, Y gc.collect() num_train = train_x.shape[0] F_buffer = torch.randn(num_train, opt.bit) G_buffer = torch.randn(num_train, opt.bit) if opt.use_gpu: train_L = train_L.cuda() F_buffer = F_buffer.cuda() G_buffer = G_buffer.cuda() Sim = calc_neighbor(train_L, train_L) B = torch.sign(F_buffer + G_buffer) batch_size = opt.batch_size lr = opt.lr optimizer_img = SGD(img_model.parameters(), lr=lr) optimizer_txt = SGD(txt_model.parameters(), lr=lr) learning_rate = np.linspace(opt.lr, np.power(10, -6.), opt.max_epoch + 1) result = {'loss': []} ones = torch.ones(batch_size, 1) ones_ = torch.ones(num_train - batch_size, 1) unupdated_size = num_train - batch_size max_mapi2t = max_mapt2i = 0. for epoch in range(opt.max_epoch): # train image net for i in tqdm(range(num_train // batch_size)): index = np.random.permutation(num_train) ind = index[0:batch_size] unupdated_ind = np.setdiff1d(range(num_train), ind) sample_L = Variable(train_L[ind, :]) image = Variable(train_x[ind].type(torch.float)) if opt.use_gpu: image = image.cuda() sample_L = sample_L.cuda() ones = ones.cuda() ones_ = ones_.cuda() # similar matrix size: (batch_size, num_train) S = calc_neighbor(sample_L, train_L) # S: (batch_size, num_train) cur_f = img_model(image) # cur_f: (batch_size, bit) F_buffer[ind, :] = cur_f.data F = Variable(F_buffer) G = Variable(G_buffer) theta_x = 1.0 / 2 * torch.matmul(cur_f, G.t()) logloss_x = -torch.sum(S * theta_x - torch.log(1.0 + torch.exp(theta_x))) quantization_x = torch.sum(torch.pow(B[ind, :] - cur_f, 2)) balance_x = torch.sum( torch.pow(cur_f.t().mm(ones) + F[unupdated_ind].t().mm(ones_), 2)) # intra loss XXX: added theta_x_intra = 1.0 / 2 * torch.matmul(cur_f, F.t()) logloss_x_intra = -torch.sum(S * theta_x_intra - Logtrick(theta_x_intra, opt.use_gpu)) loss_x = logloss_x + opt.gamma * quantization_x + opt.eta * balance_x + logloss_x_intra #loss_x = logloss_x + opt.gamma * quantization_x + opt.eta * balance_x loss_x /= (batch_size * num_train) optimizer_img.zero_grad() loss_x.backward() optimizer_img.step() # train txt net for i in tqdm(range(num_train // batch_size)): index = np.random.permutation(num_train) ind = index[0:batch_size] unupdated_ind = np.setdiff1d(range(num_train), ind) sample_L = Variable(train_L[ind, :]) text = train_y[ind, :].unsqueeze(1).unsqueeze(-1).type(torch.float) text = Variable(text) if opt.use_gpu: text = text.cuda() sample_L = sample_L.cuda() # similar matrix size: (batch_size, num_train) S = calc_neighbor(sample_L, train_L) # S: (batch_size, num_train) cur_g = txt_model(text) # cur_f: (batch_size, bit) G_buffer[ind, :] = cur_g.data F = Variable(F_buffer) G = Variable(G_buffer) # calculate loss # theta_y: (batch_size, num_train) theta_y = 1.0 / 2 * torch.matmul(cur_g, F.t()) logloss_y = -torch.sum(S * theta_y - torch.log(1.0 + torch.exp(theta_y))) quantization_y = torch.sum(torch.pow(B[ind, :] - cur_g, 2)) balance_y = torch.sum( torch.pow(cur_g.t().mm(ones) + G[unupdated_ind].t().mm(ones_), 2)) # intra loss XXX: theta_y_intra = 1.0 / 2 * torch.matmul(cur_g, G.t()) logloss_y_intra = -torch.sum(S * theta_y_intra - Logtrick(theta_y_intra, opt.use_gpu)) loss_y = logloss_y + opt.gamma * quantization_y + opt.eta * balance_y + logloss_y_intra #loss_y = logloss_y + opt.gamma * quantization_y + opt.eta * balance_y loss_y /= (num_train * batch_size) optimizer_txt.zero_grad() loss_y.backward() optimizer_txt.step() # update B B = torch.sign(F_buffer + G_buffer) # calculate total loss loss = calc_loss(B, F, G, Variable(Sim), opt.gamma, opt.eta) print('...epoch: %3d, loss: %3.3f, lr: %f' % (epoch + 1, loss.data, lr)) result['loss'].append(float(loss.data)) if opt.valid: mapi2t, mapt2i = valid(img_model, txt_model, query_x, retrieval_x, query_y, retrieval_y, query_L, retrieval_L) print( '...epoch: %3d, valid MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (epoch + 1, mapi2t, mapt2i)) if mapt2i >= max_mapt2i and mapi2t >= max_mapi2t: max_mapi2t = mapi2t max_mapt2i = mapt2i img_model.save(img_model.module_name + '.pth') txt_model.save(txt_model.module_name + '.pth') lr = learning_rate[epoch + 1] # set learning rate for param in optimizer_img.param_groups: param['lr'] = lr for param in optimizer_txt.param_groups: param['lr'] = lr print('...training procedure finish') if opt.valid: print(' max MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (max_mapi2t, max_mapt2i)) result['mapi2t'] = max_mapi2t result['mapt2i'] = max_mapt2i else: mapi2t, mapt2i = valid(img_model, txt_model, query_x, retrieval_x, query_y, retrieval_y, query_L, retrieval_L) print(' max MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (mapi2t, mapt2i)) result['mapi2t'] = mapi2t result['mapt2i'] = mapt2i write_result(result)
print ('word len\t:\t{}'.format(word_len)) print ('word dim\t:\t{}'.format(word_dim)) print ('# of hidden\t:\t{}'.format(n_units)) print ('# of output\t:\t{}'.format(n_label)) print ('filter size\t:\t{}x{}x{}'.format(filter_length, filter_width, filter_height)) print ('batch size\t:\t{}'.format(batch_size)) ###load oldmodel model = cntn.CNTN(output_channel, filter_length, filter_width, filter_height, n_units, n_label) cf = L.Classifier(model) optimizer = optimizers.Adam() optimizer.setup(cf) ###load dataset training = open(train_url, 'r', encoding='utf-8').readlines() dataset = dh.load_data( training, doc_len, word_len, token='##') x_train = dataset['source'] keyword_train = dataset['keyword'] y_train = dataset['target'] docs = dataset['docs'] words = dataset['words'] N = len(training) print ('# of training\t:\t{}'.format(N)) ###gpu setting if gpu == -1: xp = np print ('###\tUsing CPU') else:
def eval_nested_kfold(encoder, name, k=10, seed=1234, verbose=False, s=16): """ Evaluate features with nested K-fold cross validation Outer loop: Held-out evaluation Inner loop: Hyperparameter tuning Datasets can be found at http://nlp.stanford.edu/~sidaw/home/projects:nbsvm Options for name are 'MR', 'CR', 'SUBJ' and 'MPQA' """ # Load the dataset and extract features z, features = data_handler.load_data(encoder, name, seed=seed) scan = [2 ** t for t in range(2, 8, 1)] kf = KFold(k, random_state=seed) scores = [] for train, test in kf.split(features): # Split data X_train = features[train] y_train = z['labels'][train] X_test = features[test] y_test = z['labels'][test] # def inner_kfold(C): # # Inner KFold # innerkf = KFold(k, random_state=seed + 1) # innerscores = [] # for innertrain, innertest in innerkf.split(X_train): # # # Split data # X_innertrain = X_train[innertrain] # y_innertrain = y_train[innertrain] # X_innertest = X_train[innertest] # y_innertest = y_train[innertest] # # # Train classifier # clf = LogisticRegression(C=C) # clf.fit(X_innertrain, y_innertrain) # acc = clf.score(X_innertest, y_innertest) # innerscores.append(acc) # # return np.mean(innerscores) # # scanscores = Parallel(len(scan))(delayed(inner_kfold)(s) for s in scan) # # Get the index of the best score # s_ind = np.argmax(scanscores) # s = scan[int(s_ind)] # Train classifier clf = LogisticRegression(C=s) clf.fit(X_train, y_train) # Evaluate acc = clf.score(X_test, y_test) scores.append(acc) if verbose: print(s, acc) score = np.mean(scores) print(score) return score