def errhist(ax, cat, prefeat, trufeat, normrad=3.0, **kwargs): """ Shows the distribution of prediction errors (prefeat - trufeat), as a histogram. If prefeat has an errcolname, each term is normalized by this errobar. """ if trufeat.errcolname is not None: logger.warning("The 'truth' has an uncertainty, this is strange. Inverted arguments?") data = utils.getdata(cat, [prefeat, trufeat]) # Good to do this here (and not leave it only to hist), so that data has a reduced number # of columns. Indeed we add some columns below. if prefeat.errcolname is None: # Then we plot a simple histogram of the prediction errors data["err"] = data[prefeat.colname] - data[trufeat.colname] err = tools.feature.Feature("err", nicename = "%s - %s" % (prefeat.nicename, trufeat.nicename)) hist(ax, data, err, normed=True, **kwargs) else: # We normalize the residuals with the uncertainties data["err"] = (data[prefeat.colname] - data[trufeat.colname]) / data[prefeat.errcolname] err = tools.feature.Feature("err", -normrad, normrad, nicename = "(%s - %s) / %s" % (prefeat.colname, trufeat.colname, prefeat.errcolname)) hist(ax, data, err, normed=True, label = "Residuals normalized by uncertainty", **kwargs) x = np.linspace(-normrad, normrad, 1000) ax.plot(x, norm.pdf(x, 0.0, 1.0), color="black", label="Standard normal distribution")
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--model_name", default='all', type=str, help="Enter model name to run or enter 'all' to run all models in chain" ) parser.add_argument("--data_dir", type=str, default='../data/processed') parser.add_argument("--output_dir", type=str, default='../tuned_results') parser.add_argument("--input_filename", type=str, default='trade_savez_files.npz') parser.add_argument("-tune", type=list, default=[], help="List all paramaters to tune with defaults") # parser.add_argument("-with_defaults", type = int, default = 10) parser.add_argument("-epochs", type=int, default=10) parser.add_argument( "-lr_list", type=list, default=[], help="Specify range of learning rates in list for tuning") parser.add_argument( "-weight_decay_list", type=list, default=[], help="Specify range of weight decay in list for tuning") args = parser.parse_args() input_file_path = os.path.join(args.data_dir, args.input_filename) input_data = getdata(input_file_path) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("Running classifiers") hparams = dict(model_name=args.model_name, data_dir=args.data_dir, output_dir=args.output_dir, input_filename=args.input_filename, epochs=args.epochs, lr=0.001, weight_decay=5e-3) tparams = dict() if args.tune: tparams = init_params(args.tune) else: tparams = init_params() if args.lr_list: tparams['lr'] = args.lr_list if args.weight_decay_list: tparams['weight_decay'] = args.weight_decay_list best_acc, best_params = tuner(hparams, tparams)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--model_name", type=str, default="all") parser.add_argument("--data_dir", type=str, default="../data/processed") parser.add_argument("--output_dir", type=str, default="../results") parser.add_argument("-input_filename", type=str, default="trade_savez_files.npz") parser.add_argument("-lr", type=float, default=0.001, help="Enter learning rate for training") parser.add_argument("-weight_decay", type=float, default=0.005, help="Enter weight decay rate") parser.add_argument("-epochs", default=10, help="Input number of training epochs") args = parser.parse_args() input_file_path = os.path.join(args.data_dir, args.input_filename) input_data = getdata(input_file_path) print("Running link prediction...") if args.model_name == 'all': run_GAE(input_data, args.output_dir, epochs=args.epochs, lr=args.lr, weight_decay=args.weight_decay) run_VGAE(input_data, args.output_dir, epochs=args.epochs, lr=args.lr, weight_decay=args.weight_decay) elif args.model_name == 'GAE': run_GAE(input_file_path, args.output_dir, epochs=args.epochs, lr=args.lr, weight_decay=args.weight_decay) elif args.model_name == 'VGAE': run_VGAE(input_data, args.output_dir, epochs=args.epochs, lr=args.lr, weight_decay=args.weight_decay)
def main(): trainingparams = [0.01, 0.02, 0.03, 0.125, 0.625, 1] iterations = 5 eta = 0.001 epsilon = 0.001 valuerange = 10 params = [] data = utils.getdata('breast-cancer-wisconsin.data.txt') nb = naivebayes.naivebayes(data, trainingparams, iterations, valuerange) nb['label'] = 'Naive Bayes' lr = logisticregression.logisticregression(data, trainingparams, iterations, eta, epsilon) lr['label'] = 'Logistic Regression' params.append(lr) params.append(nb) plot = graph.plot(params, 'Training Set', 'Accuracy') plot.show() return
assert (upper_half(0, 63)) == (32, 63) assert (lower_half(32, 63)) == (32, 47) assert (upper_half(32, 47)) == (40, 47) assert (upper_half(40, 47)) == (44, 47) assert (lower_half(44, 47)) == (44, 45) assert (lower_half(44, 45)) == (44, 44) assert (upper_half(0, 7)) == (4, 7) assert (lower_half(4, 7)) == (4, 5) assert (upper_half(4, 5)) == (5, 5) print(find_seat("FBFBBFFRLR")) assert find_seat("FBFBBFFRLR") == 357 if __name__ == "__main__": tests() print("day 05") all_seats = list(range(50, 1024)) max_seat = -1 for line in utils.getdata("data/d5.txt"): seat = find_seat(line) print("##### ", line, " => ", seat) max_seat = max(seat, max_seat) all_seats.remove(seat) print("MAX : ", max_seat) print("all seats left") print(all_seats)
plt.rcParams['figure.figsize'] = (20.0, 10.0) plot = True if (len(sys.argv) > 1): if (sys.argv[1] == 'n'): plot = False if plot: plt.ion() else: plt.ioff() import theano import theano.tensor as T rng = np.random label = '3newap' X, Y = utils.getdata(label, -1) print X.shape, Y.shape nobj = 3 ind = np.array([[0, 1, 6, 7, 8, 9, 10, 11], [2, 3, 12, 13, 14, 15, 16, 17], [4, 5, 18, 19, 20, 21, 22, 23]]) nfeats4obj = ind.shape[1] newX = [] newY = [] for i in xrange(nobj): tY = Y[:, 2 * i:2 * i + 2] tX = X[:, ind[i]] for j in xrange(nobj): if j != i: tX = np.concatenate((tX, X[:, ind[i]], X[:, ind[j]]), axis=1)
plt.ion() plot = True if (len(sys.argv) > 1): if (sys.argv[1] == 'n'): plot = False import theano import theano.tensor as T rng = np.random import utilstheano import utils import feature_selection label = '2aprilr' X, Y = utils.getdata(label, 300) feats = X.shape[1] nout = 1 # Declare Theano symbolic variables x = T.matrix("x") y = T.dvector("y") numhidden = [feats, 30] wh = [] bh = [] o = x w = theano.shared(rng.randn(numhidden[-1], 1) / np.sqrt(numhidden[-1]), name="w") b = theano.shared(np.zeros(1), name="b")
#! /usr/bin/env python # -*- coding: utf-8 -*- import utils, sys, re ########### def tests(): pass ########### if __name__ == "__main__": tests() print("day 07") for line in utils.getdata("data/d7_ex.txt"): print line print("final shit")
# -*- coding: utf-8 -*- import numpy as np import pandas as pd import seaborn as sns; sns.set() import matplotlib.pyplot as plt import sklearn.model_selection as model_selection from sklearn.ensemble import RandomForestClassifier from sklearn.preprocessing import LabelEncoder from sklearn import svm from sklearn import metrics import constants import utils ## BINARY CLASSIFICATION (RANDOM FOREST) wine_data = utils.getdata('data/winequality-red.csv') wine_data['quality_binary'] = pd.cut(wine_data['quality'], bins=constants.BINARY_BINS, labels=constants.BGROUP_NAMES) quality_binary = LabelEncoder() wine_data['quality_binary'] = quality_binary.fit_transform( wine_data['quality_binary']) X = wine_data.drop(columns=["quality"]) y = pd.Series.to_frame(wine_data['quality_binary']) X_train, X_test, y_train, y_test = model_selection.train_test_split( X, y, test_size=0.3, random_state=123) rf_model = RandomForestClassifier(n_estimators=1000, criterion="entropy", max_features=3, min_impurity_decrease=0.0, min_samples_split=3, min_samples_leaf=2, oob_score=True, random_state=constants.SEED, class_weight=None, max_depth=None) rf_model.fit(X_train, y_train)
def do_train(model_name, data_dir, output_dir, input_filename, epochs, lr, weight_decay): input_file_path = os.path.join(data_dir, input_filename) input_data = getdata(input_file_path) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("Running classifiers")
alexnet.add(BatchNormalization()) alexnet.add(Activation('relu')) alexnet.add(Dropout(0.5)) # Layer 8 alexnet.add(Dense(n_classes)) alexnet.add(BatchNormalization()) alexnet.add(Activation('softmax')) return alexnet # Create AlexNet model model = alexnet_model() data_train, train_y, data_test, test_y = getdata() """print(data_train.shape) print(data_train[0].shape) print(data_train[0][0].shape) print(data_train[0][0].shape) print(train_y.shape) print(train_y[0].shape) print(train_y[0][0].shape)""" model.summary() model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.RMSprop(lr=0.01), metrics=['accuracy']) #model.summary()
#! /usr/bin/env python # -*- coding: utf-8 -*- import utils, sys def slope(mymap, h, v): ph = 0 pv = 0 keep = True tree = 0 while keep: ph = (ph + h) % len(mymap[0]) pv += v if pv >= len(mymap): keep = False return tree if mymap[pv][ph] == "#": tree += 1 if __name__ == "__main__": print("day 03") mymap = [line for line in utils.getdata("data/d3.txt")] for (i, j) in [(3, 1), (1, 1), (5, 1), (7, 1), (1, 2)]: tr = slope(mymap, i, j) print("slope: ", i, "x", j, "=", tr)
# coding: utf-8 import keras from keras.layers import Dense, Conv2D, BatchNormalization, Activation from keras.layers import AveragePooling2D, MaxPooling2D, Input, Flatten from keras.optimizers import Adam from keras.regularizers import l2 from keras import backend as K from keras.models import Model, load_model from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau from utils import plot_model, getdata import numpy as np import tensorflow as tf from classification_models.classification_models.resnet import ResNet18, preprocess_input data_train, box_train, data_test, box_test = getdata() # metric function def my_metric(labels, predictions): threshhold = 0.75 x = predictions[:, 0] * 224 x = tf.maximum(tf.minimum(x, 224.0), 0.0) y = predictions[:, 1] * 224 y = tf.maximum(tf.minimum(y, 224.0), 0.0) width = predictions[:, 2] * 224 width = tf.maximum(tf.minimum(width, 224.0), 0.0) height = predictions[:, 3] * 224 height = tf.maximum(tf.minimum(height, 224.0), 0.0) label_x = labels[:, 0] label_y = labels[:, 1]
def validatepass_pos(chain): rule, passw = chain.split(':') motiv, letter = rule.strip().split(' ') p1, p2 = motiv.split('-') #cl = passw.count(letter) b = ((passw)[int(p1)] == letter) ^ ((passw)[int(p2)] == letter) print("p1 ", p1, " p2 ", p2, " let ", letter, " l1 ", (" " + passw)[int(p1)], " l2 ", (" " + passw)[int(p2)], " b ", b) return b if __name__ == "__main__": print("day 02") countok = 0 countok2 = 0 count = 0 validatepass_pos("1-3 a: abcde") validatepass_pos("1-3 b: cdefg") validatepass_pos("2-9 c: ccccccccc") for i in utils.getdata("data/d2.txt"): print(i) count += 1 if validatepass_count(i): countok += 1 if validatepass_pos(i): countok2 += 1 print("count ", count, " ok ", countok, "ok2", countok2)