def main(): args = get_args() logging.info("Extracting domains and evaluation results for all models") rescaled_domain_lst, domain_name_lst, eval_lst, BEST, WORST = \ preprocess.extract_data(args.modeldir, args.architecture, args.rnn_cell_type, args.metric, args.best) embedding_lst = get_embedding(args, rescaled_domain_lst, domain_name_lst, eval_lst) kernel = get_kernel.get_kernel(args.architecture, args.embedding, args.embedding_distance, args.kernel, domain_name_lst, len(embedding_lst[0]), args.weight) if args.embedding == "mds": D = kernel.get_value(embedding_lst) D = np.exp(-np.array(D)) embedding_lst, S = get_mds_embedding.mds(D) kernel = get_kernel.get_kernel(args.architecture, "origin", args.embedding_distance, "logsquared", domain_name_lst, len(embedding_lst[0]), args.weight) logging.info("kernel: ", args.kernel) logging.info(kernel.get_value(np.atleast_2d(rescaled_domain_lst))) def objective_function(x): ''' Map a sampled domain to evaluation results returned from the model :param x: domain sampled from bayesian optimization :return: the corresponding evaluation result ''' for i in range(len(embedding_lst)): if (x == embedding_lst[i]).all(): return eval_lst[i] results, best_ind, fix_budget, close_best = run_bayesian_optimization(args, kernel, objective_function, embedding_lst, eval_lst, BEST, WORST) write_results(args, results, best_ind, fix_budget, close_best)
def main(): args = get_args() # 0. Prepare data logger.info("Extracting data ...") rescaled_domain_lst, domain_name_lst, eval_lst, BEST, WORST = \ preprocess.extract_data(args.modeldir, args.architecture, args.rnn_cell_type, args.metric, args.best) logger.info("Best point: {0}".format(BEST)) X = np.array(rescaled_domain_lst) Y = np.array(eval_lst) best_ind = [] fix_budget = [] close_best = [] for nr in range(args.num_run): # 1. setup logger logdir = args.output + "_log" if not os.path.isdir(logdir): os.mkdir(logdir) file_handler = logging.FileHandler(logdir + "/" + str(nr) + ".log") file_handler.setLevel(logging.INFO) logger.addHandler(file_handler) # 2. Initialization if nr < X.shape[0] - 2: start = nr end = nr + 3 else: start = nr % (X.shape[0] - 2) end = nr % (X.shape[0] - 2) + 3 ind_label = np.arange(start, end) num_label = 3 # 3. Find the best label logger.info("Building the graph ...") graph_obj = graph.Graph(args.model, X, Y, args.distance, args.sparsity, logger, domain_name_lst, args.distance_param, args.k, ind_label, num_label) num_update = 1 while True: logger.info("###############################") logger.info("Update # {0}: ".format(num_update)) graph_obj.update() if BEST in graph_obj.y_label: logger.info( "Found the best configuration at update # {0}".format( num_update)) break num_update += 1 # 4. Write stats best_ind.append(graph_obj.y_label.shape[0] - 3) fix_budget.append(abs(max(graph_obj.y_label[:args.budget]) - BEST)) for i in range(graph_obj.y_label.shape[0]): if abs(graph_obj.y_label[i] - BEST) <= args.dif: close_best.append(i + 1) break write_results(args, best_ind, fix_budget, close_best, graph_obj)
parser.add_argument('--kernel', default="expsquared", type=str, choices=["constant", "polynomial", "linear", "dotproduct", "exp", "expsquared", "matern32", "matern52", "rationalquadratic", "expsine2", "heuristic"], help='Specify the kernel for Gaussian process.') # Other arguments parser.add_argument("--threshold", type=int, default=5, help="Remove samples with bad performance(BLEU).") parser.add_argument("--output", help="Output directory.") return parser.parse_args() if __name__ == '__main__': args = get_args() modeldir = "/export/a10/kduh/p/mt/gridsearch/" + args.dataset + "/models/" x, y, _ = extract_data(modeldir=modeldir, threshold=args.threshold, architecture=args.architecture, rnn_cell_type=args.rnn_cell_type) y = -y lower = np.zeros((x.shape[1])) upper = np.ones((x.shape[1])) if args.architecture == "trans": domain_name_lst = ['num_layers', 'transformer_attention_heads', 'transformer_feed_forward_num_hidden', 'transformer_model_size', 'bpe_symbols', 'initial_learning_rate', 'num_embed'] else: domain_name_lst = [] kernel = get_kernel.get_kernel(args.architecture, args.kernel, domain_name_lst, x.shape[1]) def objective_function(sample): ''' Map a sampled domain to evaluation results returned from the model :param x_sample: domain sampled from bayesian optimization
import numpy as np import preprocess as pp from keras.models import model_from_json question1, question2 = pp.extract_data("quora-question-pairs/test.csv", 'test') question1_word_sequences, question2_word_sequences, word_index = pp.tokenize( question1, question2) embeddings_index = pp.get_embeddings("glove.840B.300d/glove.840B.300d.txt") nb_words, word_embedding_matrix = pp.get_embedding_matrix( word_index, embeddings_index) q1_data, q2_data, word_embedding_matrix, nb_words = pp.process_data( question1_word_sequences, question2_word_sequences, word_embedding_matrix, nb_words, 'test') X_train = np.stack((q1_data, q2_data), axis=1) Q1_train = X_train[:, 0] Q2_train = X_train[:, 1] json_file = open('best_weights/model.json', 'r') loaded_model_json = json_file.read() json_file.close() model = model_from_json(loaded_model_json) model.load_weights("best_weights/weights.h5") print("Loaded model from disk") model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) score = model.predict([Q1_train, Q2_train]) print(score)
import sys sys.path.insert(1, '/export/a08/xzhan138/Auto-tuning/multi-objective/regressor') sys.path.insert(1, '/export/a08/xzhan138/Auto-tuning/multi-objective') import pickle import numpy as np import random from gp import GP from krr import KRR from gbssl import GBSSL from preprocess import extract_data if __name__ == "__main__": modeldir = "/export/a10/kduh/p/mt/gridsearch/robust19-ja-en/models/" x, y, _ = extract_data(modeldir, 5) output = "/export/a08/xzhan138/Auto-tuning/diagnosis_output/mse{0}.pkl" models = [KRR, GP, GBSSL] for i in random.sample(range(len(y) - 3), 5): print("init: {0}".format(i)) mse_dic = {} for m in range(len(models)): mse = [] label_ids = np.array([i, i + 1, i + 2]) model = models[m] print("model: {0}".format(model.__name__)) while len(label_ids) != len(y): opt_model = model(x, y[label_ids], label_ids) y_preds, y_vars = opt_model.fit_predict() del opt_model unlabel_ids = np.array(
return best - current_best if __name__ == "__main__": architectures = ["rnn", "trans"] evaluations = [s2best, s2close_best, ns2dif_best] input = '/export/a08/xzhan138/Auto-tuning/single_output/' for architecture in architectures: arch_dir = input + architecture if os.path.isdir(arch_dir): for dataset_name in os.listdir(arch_dir): dataset_dir = arch_dir + "/" + dataset_name if os.path.isdir(dataset_dir): evals = defaultdict(list) modeldir = "/export/a10/kduh/p/mt/gridsearch/" + dataset_name + "/models/" for pf in os.listdir(dataset_dir): if "gru" in pf: x,y,_ = extract_data(modeldir, 5, architecture, "gru") else: x,y,_ = extract_data(modeldir, 5, architecture) best_id = np.argmax(y) if pf.endswith("pkl"): with open(pf, 'rb') as f: I = pickle.load(f) for eval in evaluations: I_eval = np.apply_along_axis(eval, 1, I, best_id=best_id, y=y) evals[pf[:-4]].append((np.average(I_eval), np.std(I_eval))) with open(dataset_dir + "/eval.pkl", 'wb') as f: pickle.dump(evals)
import classifier from preprocess import extract_data timesteps = 140 clf = classifier.Classifier(timesteps) train_file = '../../tsv/train.tsv' test_file = '../../tsv/test.tsv' X_train, y_train = extract_data(train_file, timesteps) X_test, y_test = extract_data(test_file, timesteps) clf.fit_classif(X_train, y_train) clf.evaluate(X_test, y_test)
def main(): # load and preprocess data train_labels, train_imgs = extract_data(config.train_path) test_labels, test_imgs = extract_data(config.test_path) f = open(config.output_file, 'w') # model selection if config.select_model is True: print("Selecting model...") f.write("Scores for model selection:\n") # original image plain = True # set parameters for feature extraction pool = {'take': False, 'class': 'max'} hist = {'take': False, 'h': [4], 'w': [4]} grad = {'take': False, 'class': 'hist'} chain = {'take': False, 'class': 'hist'} select_feats1 = get_feats(train_imgs, plain, pool, hist, grad, chain) # feature vector pool = {'take': False, 'class': 'max'} hist = {'take': True, 'h': [4], 'w': [4]} grad = {'take': True, 'class': 'hist'} chain = {'take': True, 'class': 'hist'} select_feats2 = get_feats(train_imgs, plain, pool, hist, grad, chain) # get cross-validation scores f.write("Baseline (original image):" + '\n') print("logistic regression models:") scores = cross_valid(config.models_select1, select_feats1, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names1[i] + ':' + str(scores[i]) + '\n') print("multi-class logistic regression models:") scores = cross_valid(config.models_select2, select_feats1, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names2[i] + ':' + str(scores[i]) + '\n') print("k-nearest neighbour models:") scores = cross_valid(config.models_select3, select_feats1, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names3[i] + ':' + str(scores[i]) + '\n') f.write("\nFeature vector:" + '\n') scores = cross_valid(config.models_select1, select_feats2, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names1[i] + ':' + str(scores[i]) + '\n') print("multi-class logistic regression models:") scores = cross_valid(config.models_select2, select_feats2, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names2[i] + ':' + str(scores[i]) + '\n') print("k-nearest neighbour models:") scores = cross_valid(config.models_select3, select_feats2, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names3[i] + ':' + str(scores[i]) + '\n') f.write("\n######################\n\n") # feature selection if config.select_feature is True: print("Selecting features...") f.write("Scores for feature selection:\n") plain = True # set parameters for feature extraction pool = {'take': False, 'class': 'max'} hist = {'take': False, 'h': [4], 'w': [4]} grad = {'take': False, 'class': 'hist'} chain = {'take': False, 'class': 'hist'} # histogram hist['take'] = True print("Extract histogram from training data set...") f.write("\nHistogram:\n") select_feats = get_feats(train_imgs, plain, pool, hist, grad, chain) scores = cross_valid(config.models, select_feats, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names[i] + ':' + str(scores[i]) + '\n') # gradient histogram hist['take'] = False grad['take'] = True print("Extract gradient histogram from training data set...") f.write("\nGradient histogram:\n") select_feats = get_feats(train_imgs, plain, pool, hist, grad, chain) scores = cross_valid(config.models, select_feats, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names[i] + ':' + str(scores[i]) + '\n') # gradient image grad['class'] = 'plain' print("Extract gradient image from training data set...") f.write("\nGradient image:\n") select_feats = get_feats(train_imgs, plain, pool, hist, grad, chain) scores = cross_valid(config.models, select_feats, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names[i] + ':' + str(scores[i]) + '\n') # chain code histogram grad['take'] = False chain['take'] = True print("Extract chain code histogram from training data set...") f.write("\nChain code histogram:\n") select_feats = get_feats(train_imgs, plain, pool, hist, grad, chain) scores = cross_valid(config.models, select_feats, train_labels) print(scores) for i, s in enumerate(scores): f.write(config.names[i] + ':' + str(scores[i]) + '\n') f.write("\n######################\n\n") if config.produce_results is True or config.draw_ROC is True: # feature extraction print("Extract feature from training data set...") train_feats = get_feats(train_imgs, config.plain, config.pool, config.hist, config.grad, config.chain) print("Extract feature from testing data set...") test_feats = get_feats(test_imgs, config.plain, config.pool, config.hist, config.grad, config.chain) print("All data processed. Number of features extracted is " + str(len(train_feats[0]))) if config.produce_results is True: print("Producing prediction results...") f.write("Prediction results\n") f.write('original image: ' + str(config.plain)) f.write('\n') f.write('pooled:' + str(config.pool)) f.write('\n') f.write('histogram:' + str(config.hist)) f.write('\n') f.write('gradient:' + str(config.grad)) f.write('\n') f.write('chain code:' + str(config.chain)) f.write('\n\n') all_preds = final_result(config.models, config.names, train_feats, train_labels, test_feats, test_labels, f) if config.visualize_error is True: preds = all_preds[2] err_imgs = test_imgs[preds != test_labels] err_labels = test_labels[preds != test_labels] visualize(err_labels, err_imgs) if config.draw_ROC is True: print("Drawing ROC for LDA model...") preds_proba = plot_ROC(config.lda, train_feats, train_labels, test_feats, test_labels)
import numpy as np import preprocess as pp from keras.models import * from keras_model import model question1, question2, is_duplicate = pp.extract_data( "quora-question-pairs/train.csv", 'train') question1_word_sequences, question2_word_sequences, word_index = pp.tokenize( question1, question2) embeddings_index = pp.get_embeddings("glove.840B.300d/glove.840B.300d.txt") nb_words, word_embedding_matrix = pp.get_embedding_matrix( word_index, embeddings_index) q1_data, q2_data, labels, word_embedding_matrix, nb_words = pp.process_data( question1_word_sequences, question2_word_sequences, word_embedding_matrix, nb_words, 'train', is_duplicate) X_train = np.stack((q1_data, q2_data), axis=1) y_train = labels Q1_train = X_train[:, 0] Q2_train = X_train[:, 1] model = model(nb_words, word_embedding_matrix) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit([Q1_train, Q2_train], y_train, batch_size=32, epochs=100) model_json = model.to_json() with open("best_weights/model.json", "w") as json_file: