def mainregress(selection, alpha): if len(selection) < 2: return x = xdown.get()['value'] y = ydown.get()['value'] tabdata = [] mldatax = [] mldatay = [] species = iris.Species.unique() for i, p in enumerate(selection['points']): mldatax.append(p['x']) mldatay.append(p['y']) tabdata.append({ x: p['x'], y: p['y'], 'species': species[p['curve']] }) X = np.c_[mldatax, np.array(mldatax) ** 2] ridge = KernelRidge(alpha=alpha).fit(X, mldatay) xspace = np.linspace(min(mldatax)-1, max(mldatax)+1, 100) plot = pw.scatter(mldatax, mldatay, label='train', markersize=15) for i, df in iris.groupby('Species'): plot += pw.scatter(df[x], df[y], label=i) plot += pw.line(xspace, ridge.predict(np.c_[xspace, xspace**2]), label='model', mode='lines') plot.xlabel = x plot.ylabel = y linear.do_all(plot.dict) table1.do_data(pd.DataFrame(tabdata))
def test_kernel_ridge_singular_kernel(): # alpha=0 causes a LinAlgError in computing the dual coefficients, # which causes a fallback to a lstsq solver. This is tested here. pred = Ridge(alpha=0, fit_intercept=False).fit(X, y).predict(X) kr = KernelRidge(kernel="linear", alpha=0) ignore_warnings(kr.fit)(X, y) pred2 = kr.predict(X) assert_array_almost_equal(pred, pred2)
def fit( self, x , y , size=1 ): self.model = Sequential() self.model.add(Dense( int( embeddings_dim / 2.0 ) , input_dim=embeddings_dim , init='uniform' , activation='tanh')) self.model.add(Dense( int( embeddings_dim / 4.0 ) , init='uniform' , activation='tanh')) self.model.add(Dense(size , init='uniform' ) ) self.model.compile(loss='mse', optimizer='rmsprop') self.model = KernelRidge( kernel='rbf' ) self.model.fit( x , y )
def train_kernelRidgeModel(X, y, alpha=1, kernel="linear", gamma=None, degree=3, coef0=1, kernel_params=None): """ Train a kernel ridge regression model """ model = KernelRidge( alpha=alpha, kernel=kernel, gamma=gamma, degree=degree, coef0=coef0, kernel_params=kernel_params ) model = model.fit(X, y) return model
def lgo_sklearn(X,y, groups, regparam): logo = LeaveOneGroupOut() errors = [] for train, test in logo.split(X, y, groups=groups): rls = KernelRidge(kernel="rbf", gamma=0.01) rls.fit(X[train], y[train]) p = rls.predict(X[test]) e = sqerror(y[test], p) errors.append(e) return np.mean(errors)
def lpo_sklearn(X,y, regparam): lpo = LeavePOut(p=2) preda = [] predb = [] for train, test in lpo.split(X): rls = KernelRidge(kernel="rbf", gamma=0.01) rls.fit(X[train], y[train]) p = rls.predict(X[test]) preda.append(p[0]) predb.append(p[1]) return preda, predb
def ANM_causation_score(self,train_size=0.5,independence_criterion='HSIC',metric='linear',regression_method='GP'): ''' Measure how likely a given causal direction is true Parameters ---------- train_size : Fraction of given data used to training phase independence_criterion : kruskal for Kruskal-Wallis H-test, HSIC for Hilbert-Schmidt Independence Criterion metric : linear, sigmoid, rbf, poly kernel function to compute gramm matrix for HSIC gaussian kernel is used in : Nonlinear causal discovery with additive noise models Patrik O. Hoyer et. al Returns ------- causal_strength: A float between 0. and 1. ''' Xtrain, Xtest , Ytrain, Ytest = train_test_split(self.X, self.Y, train_size = train_size) if regression_method == 'GP': _gp = pyGPs.GPR() # specify model (GP regression) _gp.getPosterior(Xtrain, Ytrain) # fit default model (mean zero & rbf kernel) with data _gp.optimize(Xtrain, Ytrain) # optimize hyperparamters (default optimizer: single run minimize) #Forward case #_gp = KernelRidge(kernel='sigmoid',degree=3) #_gp.fit(Xtrain,Ytrain) ym, ys2, fm, fs2, lp = _gp.predict(Xtest) #_gp.plot() #errors_forward = _gp.predict(Xtest) - Ytest errors_forward = ym - Ytest else: _gp = KernelRidge(kernel='sigmoid') _gp.fit(Xtrain, Ytrain) errors_forward = _gp.predict(Xtest) - Ytest #Independence score forward_indep_pval = { 'kruskal': kruskal(errors_forward,Xtest)[1], 'HSIC': self.HilbertSchmidtNormIC(errors_forward,Xtest,metric=metric)[1] }[independence_criterion] return {'causal_strength':forward_indep_pval}
def fit(self, Ks, y, holdout=None): """Learn weights for kernel matrices or Kinterfaces. :param Ks: (``list``) of (``numpy.ndarray``) or of (``Kinterface``) to be aligned. :param y: (``numpy.ndarray``) Class labels :math:`y_i \in {-1, 1}` or regression targets. :param holdout: (``list``) List of indices to exlude from alignment. """ # Expand kernel interfaces to kernel matrices expand = lambda K: K[:, :] if isinstance(K, Kinterface) else K Hs = map(expand, Ks) # Assert correct dimensions assert Ks[0].shape[0] == len(y) # Fit MKL model if self.method in self.supervised: self.mkl_model.fit(Hs, y, holdout=holdout) else: self.mkl_model.fit(Hs) if self.low_rank: self.X = hstack(map(lambda e: sqrt(e[0]) * e[1], zip(self.mkl_model.mu, Hs))) if self.method in self.centered: self.X = center_kernel_low_rank(self.X) self.X[where(isnan(self.X))] = 0 # Fit ridge model with given lbd and MKL model self.ridge = KernelRidge(alpha=self.lbd, kernel="linear", ) # Fit ridge on the examples minus the holdout set inxs = list(set(range(Hs[0].shape[0])) - set(holdout)) self.ridge.fit(self.X[inxs], y[inxs]) self.trained = True else: # Fit ridge model with given lbd and MKL model self.ridge = KernelRidge(alpha=self.lbd, kernel=self.mkl_model, ) # Fit ridge on the examples minus the holdout set inxs = array(list(set(range(Hs[0].shape[0])) - set(holdout))) inxs = inxs.reshape((len(inxs), 1)).astype(int) self.ridge.fit(inxs, y[inxs]) self.trained = True
def xyz_kde(xyz,gamma,N_grid=100): xy = xyz[:,:-1] z = xyz[:,-1] x_edges = np.linspace(np.min(xy[:,0]),np.max(xy[:,0]),N_grid+1) y_edges = np.linspace(np.min(xy[:,1]),np.max(xy[:,1]),N_grid+1) x_centres = np.array([x_edges[b] + (x_edges[b+1]-x_edges[b])/2 for b in range(N_grid)]) y_centres = np.array([y_edges[b] + (y_edges[b+1]-y_edges[b])/2 for b in range(N_grid)]) x_grid, y_grid = np.meshgrid(x_centres,y_centres) xy_grid = np.array([np.ravel(x_grid),np.ravel(y_grid)]).T clf = KernelRidge(kernel='rbf',gamma=gamma).fit(xy,z) H = clf.predict(xy_grid).reshape(N_grid,N_grid) return H, x_grid, y_grid, gamma
def plot_kernel_ridge(X, y, gamma=0.5, alpha=0.1): # kernel (ridge) regression krr = KernelRidge(kernel="rbf", gamma=gamma, alpha=alpha); krr.fit(X,y); # predict x_plot = np.linspace(min(X), max(X), 100)[:,np.newaxis]; y_plot = krr.predict(x_plot); # plot plt.figure(figsize=(8,4.8)); plt.plot(X, y, 'or'); plt.plot(x_plot, y_plot) # plt.title(r"Gaussian Kernel ($\gamma=%0.2f, \alpha=%0.2f$)" % (gamma,alpha), fontsize=16) plt.title(r"Gaussian Kernel ($\gamma=%0.2f$)" % (gamma), fontsize=16)
def modelfitOne(train_X, train_y, test_X, yd, ImageId, FeatureName): n_clf = 1 # 拟合器 clf = KernelRidge(kernel='rbf', gamma=6e-4, alpha=2e-2) # 训练 print('-----------------开始训练...------------------') clf.fit(train_X, train_y) # 预测 print('-----------------开始预测...------------------') pred = clf.predict(test_X) predicted = np.zeros(len(FeatureName)) for i in range(len(FeatureName)): if i % 500 == 0: print('i =', i) else: pass imageID = ImageId[i] clfID = yd[FeatureName[i]] predicted[i] = pred[imageID, clfID] predicted = predicted*48.+48. return predicted
# ##Base models # - **LASSO Regression** : # # This model may be very sensitive to outliers. So we need to made it more robust on them. For that we use the sklearn's **Robustscaler()** method on pipeline lasso = make_pipeline(RobustScaler(), Lasso(alpha=0.0005, random_state=1)) # - **Elastic Net Regression** : # # again made robust to outliers ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3)) # - **Kernel Ridge Regression** : KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5) # - **Gradient Boosting Regression** : # # With **huber** loss that makes it robust to outliers # GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5) # - **XGBoost** :
def ml_param_scan(x_datafile, y_datafile, ids, testfiles, alpha_list=np.logspace(-1, -8, 8), gamma_list=np.logspace(-2, -10, 9), kernel_list=['rbf'], layer_list=[(40, 40, 40)], learning_rate_list=[0.001], sample_size=0.1, ml_method="krr"): print('model ' + ml_method) # load, split and scale data x_train, x_test, y_train, y_test, ids_train, ids_test = load_split_scale_data( x_datafile, y_datafile, ids, testfiles, sample_size) if ml_method == "krr": # Create kernel linear ridge regression object learner = GridSearchCV(KernelRidge(kernel='rbf'), n_jobs=8, cv=5, param_grid={ "alpha": alpha_list, "gamma": gamma_list, "kernel": kernel_list }, scoring='neg_mean_absolute_error') elif ml_method == "mlp": # Create Multi-Layer Perceptron object learner = GridSearchCV(MLPRegressor(hidden_layer_sizes=(40, 40, 40), max_iter=1600, alpha=0.001, learning_rate_init=0.001), n_jobs=8, cv=5, param_grid={ "alpha": alpha_list, "learning_rate_init": learning_rate_list, "hidden_layer_sizes": layer_list }, scoring='neg_mean_absolute_error') else: print("ML method unknown. Exiting.") exit(1) t_ml0 = time.time() learner.fit(x_train, y_train) t_ml1 = time.time() print("ml time", str(t_ml1 - t_ml0)) # getting best parameters learner_best = learner.best_estimator_ mae, mse, y_pred, train_y_pred, learner_best = predict_and_error( learner_best, x_test, x_train, y_test) ### OUTPUT ### write_output(learner, sample_size, ml_method, mae, mse, "param", ids_test, y_test, y_pred, ids_train, y_train, train_y_pred) return learner.best_params_
def trainModel(self, s=None, a=None): """ Trains model on given states and actions. Uses neural net or SVM based on global settings. """ states, actions = self.states[3:], self.actions[3:] #print "states.shape" #print states.shape #print "actions.shape" #print actions.shape if len(self.itr) == 0: self.itr = np.array([states.shape[0]]) else: self.itr = np.hstack((self.itr, states.shape[0])) '''if states.shape[0] > 2700.0: f = os.path.join(self.path, 'statesToValidate.npy') np.save(f, states) IPython.embed()''' fits = [] #actions = actions.ravel() self.clf = KernelRidge(alpha=1.0) self.clf.kernel = 'rbf' print "SIZE: ", states.shape self.clf.fit(states, actions) #IPython.embed() actions_pred = self.clf.predict(states) bad_state = np.zeros(actions_pred.shape[0]) for i in range(actions_pred.shape[0]): fit = LA.norm(actions_pred[i,:] - actions[i,:]) fits.append(fit) med = np.median(np.array(fits)) for fit in fits: if(fit>med): bad_state[i] = 1 IPython.embed() if self.useSHIV: self.labels = np.zeros(states.shape[0])+1.0 self.scaler = preprocessing.StandardScaler().fit(states) states_proc = self.scaler.transform(states) good_labels = bad_state == 0.0 states_g = states_proc[good_labels,:] bad_labels = bad_state == 1.0 states_b = states_proc[bad_labels,:] #IPython.embed() self.ahqp_solver_g.assembleKernel(states_g, np.zeros(states_g.shape[0])+1.0) self.ahqp_solver_b.assembleKernel(states_b, np.zeros(states_b.shape[0])+1.0) #IPython.embed() self.ahqp_solver_g.solveQP() self.ahqp_solver_b.solveQP() #score = self.clf.score(states, actions) #print score self.plot(fits, states, med)
if t == start: Xr = abs(x1 - x2) else: Xr = hstack((Xr, abs(x1 - x2))) y = zeros((nono, dt)) y[cho, :] = 1 #blurred if i == 0 and cho == 0: X = Xr Y = y else: X = hstack((X, Xr)) Y = hstack((Y, y)) clf = KernelRidge(alpha=1) clf.fit(X.T, Y.T) Wpred = dot(dot(Y, X.T), linalg.inv(dot(X, X.T) + err * eye(dot(X, X.T).shape[0]))) print('yay') #prediction state correct = 0 wrong = 0 lout = np.where(test_set[1] > 5)[0] vs = [] rot = [] blur = [] scale = [] diff = []
if varNames[variable] == 'eccentricity': predictor = predictor[~np.isinf(predictand)] predictand = predictand[~np.isinf(predictand)] # Prediction grid predictor_grid = np.linspace(np.min(predictor), np.max(predictor), 1000) #### KERNEL RIDGE REGRESSION alphaVec = [0.1, 0.01] sigmaVec = np.arange(5.0, 5.5, 0.5) if len(alphaVec) > 1 or len(sigmaVec) > 1: # Grid search of parameters param_grid = {"alpha": alphaVec, "kernel": [RBF(length_scale) for length_scale in sigmaVec]} kr = KernelRidge() kr = GridSearchCV(KernelRidge(), cv=5, param_grid=param_grid) else: # Run with pre-defined parameter set kr = KernelRidge(alpha=alphaVec[0], kernel='rbf', gamma=sigmaVec[0]) # Fit model kr.fit(predictor.reshape(-1,1), predictand.reshape(-1,1)) # Get best parameters bestAlpha_kr = kr.best_params_['alpha'] bestSigma_kr = kr.best_params_['kernel'].length_scale # Predict over grid kr_fit = kr.predict(predictor_grid.reshape(-1,1))
test_labels = [ label for ( txt , label ) in data[train_size:-1] ] tokenizer = Tokenizer(nb_words=max_features, filters=keras.preprocessing.text.base_filter(), lower=True, split=" ") tokenizer.fit_on_texts(train_texts) train_sequences = sequence.pad_sequences( tokenizer.texts_to_sequences( train_texts ) , maxlen=max_sent_len ) test_sequences = sequence.pad_sequences( tokenizer.texts_to_sequences( test_texts ) , maxlen=max_sent_len ) train_matrix = tokenizer.texts_to_matrix( train_texts ) test_matrix = tokenizer.texts_to_matrix( test_texts ) embedding_weights = np.zeros( ( max_features , embeddings_dim ) ) for word,index in tokenizer.word_index.items(): if index < max_features: try: embedding_weights[index,:] = embeddings[word] except: embedding_weights[index,:] = np.random.rand( 1 , embeddings_dim ) print ("") print ("Method = Linear ridge regression with bag-of-words features") model = KernelRidge( kernel='linear' ) model.fit( train_matrix , train_labels ) results = model.predict( test_matrix ) if not(is_geocoding): print ("RMSE = " + repr( np.sqrt(mean_squared_error( test_labels , results )) ) ) print ("MAE = " + repr( mean_absolute_error( test_labels , results ) ) ) else: print ("Mean error = " + repr( np.mean( [ geodistance( results[i] , test_labels[i] ) for i in range(results.shape[0]) ] ) ) ) print ("Median error = " + repr( np.median( [ geodistance( results[i] , test_labels[i] ) for i in range(results.shape[0]) ] ) ) ) print ("") print ("Method = MLP with bag-of-words features") np.random.seed(0) model = Sequential() model.add(Dense(embeddings_dim, input_dim=train_matrix.shape[1], init='uniform', activation='relu')) model.add(Dropout(0.25))
Y = (2 + np.sin(X) + noisesigma * np.random.randn(22, 1)) #Y[[5,10,15],0] = 2 * Y[[5,10,15],0] #Testing Set Xp = np.zeros((110,1)) Xp[:,0] = np.arange(0,11,.1) Yp = (2 + np.sin(Xp)) # Linear Regression reglr = linear_model.LinearRegression() reglr.fit(X,Y) Ylr = reglr.predict(Xp) # Kernel Ridge Regression regkr = KernelRidge(kernel='rbf', gamma=0.1,alpha=0.1) regkr.fit(X,Y) Ykr = regkr.predict(Xp) # Kernel Regression Yp1 = kernelregress(np.hstack((X,Y)),Xp,10) Yp2 = kernelregress(np.hstack((X,Y)),Xp,1) # Decision Tree Regressor min_samples_split = 3 regtree = tree.DecisionTreeRegressor(min_samples_split=min_samples_split) regtree = regtree.fit(X, Y) Ytree = regtree.predict(Xp) plt.plot(X,Y,'go',label='true')
def main(): T = 300.0 # Simulation temperature dt = 1 * units.fs # MD timestep nsteps = 100000 # MD number of steps mixing = [1,-1,0] # [1.0, -1.0, 0.3] # mixing weights for "real" and ML forces lengthscale = 0.5 # KRR Gaussian width. gamma = 1 / (2 * lengthscale**2) grid_spacing = 0.05 # mlmodel = GaussianProcess(corr='squared_exponential', # # theta0=1e-1, thetaL=1e-4, thetaU=1e+2, # theta0=1., # random_start=100, normalize=False, nugget=1.0e-2) mlmodel = KernelRidge(kernel='rbf', gamma=gamma, gammaL = gamma/4, gammaU=2*gamma, alpha=1.0e-2, variable_noise=False, max_lhood=False) anglerange = sp.arange(0, 2*sp.pi + grid_spacing, grid_spacing) X_grid = sp.array([[sp.array([x,y]) for x in anglerange] for y in anglerange]).reshape((len(anglerange)**2, 2)) ext_field = None # IgnoranceField(X_grid, y_threshold=-0.075, cutoff = 3.) # Bootstrap from initial database? uncomment # data = sp.loadtxt('phi_psi_F.csv') # # data[:,:2] -= 0.025 # fix because of old round_vector routine # mlmodel.fit(data[:,:2], data[:,2]) # ext_field.update_cost(mlmodel.X_fit_, mlmodel.y) # mlmodel.fit(X_grid, sp.zeros(len(X_grid))) # mlmodel.fit(sp.load('X_fitD.npy'), sp.load('y_fitD.npy')) # Prepare diagnostic visual effects. plt.close('all') plt.ion() fig, ax = plt.subplots(1, 2, figsize=(24, 13)) atoms = ase.io.read('myplum.xyz') with open('data.input', 'r') as file: lammpsdata = file.readlines() # Set temperature MaxwellBoltzmannDistribution(atoms, 0.5 * units.kB * T, force_temp=True) # Set total momentum to zero p = atoms.get_momenta() p -= p.sum(axis=0) / len(atoms) atoms.set_momenta(p) atoms.rescale_velocities(T) # Select MD propagator mdpropagator = Langevin(atoms, dt, T*units.kB, 1.0e-2, fixcm=True) # mdpropagator = MLVerlet(atoms, dt, T) # Zero-timestep evaluation and data files setup. print("START") pot_energy, f = calc_lammps(atoms, preloaded_data=lammpsdata) mlmodel.accumulate_data(round_vector(atoms.colvars(), precision=grid_spacing), 0.) # mlmodel.accumulate_data(round_vector(atoms.colvars(), precision=grid_spacing), pot_energy) printenergy(atoms, pot_energy) try: os.remove('atomstraj.xyz') except: pass traj = open("atomstraj.xyz", 'a') atoms.write(traj, format='extxyz') results, traj_buffer = [], [] # When in the simulation to update the ML fit -- optional. teaching_points = sp.unique((sp.linspace(0, nsteps**(1/3), nsteps/20)**3).astype('int') + 1) # MD Loop for istep in range(nsteps): # Flush Cholesky decomposition of K if istep % 1000 == 0: mlmodel.Cho_L = None mlmodel.max_lhood = False print("Dihedral angles | phi = %.3f, psi = %.3f " % (atoms.phi(), atoms.psi())) do_update = (istep % 60 == 59) t = get_time() mdpropagator.halfstep_1of2(f) print("TIMER 001 | %.3f" % (get_time() - t)) t = get_time() f, pot_energy, _ = get_all_forces(atoms, mlmodel, grid_spacing, T, extfield=ext_field, mixing=mixing, lammpsdata=lammpsdata, do_update=do_update) if do_update and mlmodel.max_lhood: mlmodel.max_lhood = False mdpropagator.halfstep_2of2(f) print("TIMER 002 | %.3f" % (get_time() - t)) # manual cooldown!!! if sp.absolute(atoms.get_kinetic_energy() / (1.5 * units.kB * atoms.get_number_of_atoms()) - T) > 100: atoms.rescale_velocities(T) printenergy(atoms, pot_energy/atoms.get_number_of_atoms(), step=istep) # if do_update: # try: # print("Lengthscale = %.3e, Noise = %.3e" % (1/(2 * mlmodel.gamma)**0.5, mlmodel.noise.mean())) # except: # print("") if istep % 60 == 59: t = get_time() if 'datasetplot' not in locals(): datasetplot = pl.Plot_datapts(ax[0], mlmodel) else: datasetplot.update() if hasattr(mlmodel, 'dual_coef_'): if 'my2dplot' not in locals(): my2dplot = pl.Plot_energy_n_point(ax[1], mlmodel, atoms.colvars().ravel()) else: my2dplot.update_prediction() my2dplot.update_current_point(atoms.colvars().ravel()) print("TIMER 003 | %.03f" % (get_time() - t)) t = get_time() fig.canvas.draw() print("TIMER 004 | %.03f" % (get_time() - t)) # fig.canvas.print_figure('current.png') t = get_time() # traj_buffer.append(atoms.copy()) # if istep % 100 == 0: # for at in traj_buffer: # atoms.write(traj, format='extxyz') # traj_buffer = [] results.append(sp.array([atoms.phi(), atoms.psi(), pot_energy])) print("TIMER 005 | %.03f" % (get_time() - t)) traj.close() print("FINISHED") sp.savetxt('results.csv', sp.array(results)) sp.savetxt('mlmodel.dual_coef_.csv', mlmodel.dual_coef_) sp.savetxt('mlmodel.X_fit_.csv', mlmodel.X_fit_) sp.savetxt('mlmodel.y.csv', mlmodel.y) calc = None return mlmodel
input_dim = 12 M = 15 alpha = 1.5848931924611107e-05 gamma = 0.14174741629268056 M = 200 alpha = 2.511886431509572e-09 gamma = 0.06579332246575682 #print(np.max(pos)) [train_set, test_set, train_ens, test_ens, train_fours, test_fours] = get_train_test(M, N, input_dim, pos, ens, fours) print(len(train_set)) print(type(train_set)) kr = KernelRidge(kernel='rbf', alpha=alpha, gamma=gamma) AlKRR = fit_quick(train_set, train_ens, alpha=alpha, gamma=gamma) AlKRR.original_train_set = train_set AlKRR.original_train_ens = train_ens # In[32]: # choose hyperparameter grids alphas = np.logspace(-20, -1, 6) gammas = np.logspace(-2, 1, 100) for n in range(len(Ms)): M = Ms[n] # split into test and training [train_set, test_set, train_ens, test_ens, train_fours,
def relationship_road_traffic_accidents(): accidents = glob.glob('accident/*.csv') acc = 0 for a in accidents: acc = ReadAccident.Accident(a) toronto_traffic = pd.read_csv('traffic/traffic-vehicle.csv') # Relationship between peak vehicle volume and # of accidents at that intersection shp_files = glob.glob('shapefiles/*.shp') shp_data_objs = [] for shp in shp_files: print(shp) shp_obj = ReadSHP.ReadSHPFile(shp, shp) shp_data_objs.append(shp_obj) data = acc.data intersec_id = {} other_xs = {} print('Running') for i in range(len(data)): long = data[i].long lat = data[i].lat fatal = data[i].fatal min_index = 0 min_dist = math.sqrt( math.pow(long - toronto_traffic.loc[0, 'Longitude'], 2) + math.pow(lat - toronto_traffic.loc[0, 'Latitude'], 2)) for j in range(1, len(toronto_traffic.index.values)): dist = math.sqrt( math.pow(long - toronto_traffic.loc[j, 'Longitude'], 2) + math.pow(lat - toronto_traffic.loc[j, 'Latitude'], 2)) if dist < min_dist: min_dist = dist min_index = j if min_index not in intersec_id: intersec_id[min_index] = 1 else: intersec_id[min_index] += 1 if min_index not in other_xs: missing_xs = [] for s in shp_data_objs: missing_xs.append( s.binary_search( toronto_traffic.loc[min_index, 'Longitude'], toronto_traffic.loc[min_index, 'Latitude'])) other_xs[min_index] = missing_xs xs = [] ys = [] for j in intersec_id: dt = [toronto_traffic.loc[j, '8 Peak Hr Vehicle Volume']] dt.extend(other_xs[j]) xs.append(dt) ys.append(intersec_id[j]) print(xs) xs = np.array(xs) ys = np.array(ys) # xs = sm.add_constant(xs) model = sm.OLS(ys, xs).fit() print(model.summary()) print(model.params) clf = KernelRidge(alpha=1.0) clf.fit(xs, ys) file = open('k_reg.pickle', 'wb') pickle.dump(clf, file) print(clf.get_params())
# Add noise to targets y[::5] += 3 * (0.5 - rng.rand(X.shape[0] // 5)) X_plot = np.linspace(0, 5, 100000)[:, np.newaxis] # ############################################################################# # Fit regression model train_size = 100 svr = GridSearchCV(SVR(kernel='rbf', gamma=0.01), cv=5, param_grid={ "C": [1e0, 1e1, 1e2, 1e3], "gamma": np.logspace(-2, 2, 5) }) kr = GridSearchCV(KernelRidge(kernel='rbf', gamma=0.1), cv=5, param_grid={ "alpha": [1e0, 1e-1, 1e-2, 1e-3], "gamma": np.logspace(-2, 2, 5) }) t0 = time.time() svr.fit(X[:train_size], y[:train_size]) svr_fit = time.time() - t0 print("SVR complexity and bandwidth selected and model fitted in %.3f s" % svr_fit) t0 = time.time() kr.fit(X[:train_size], y[:train_size]) kr_fit = time.time() - t0
# # LinearRegression # Ridge # Lasso # Random Forrest # Gradient Boosting Tree # Support Vector Regression # Linear Support Vector Regression # ElasticNet # Stochastic Gradient Descent # BayesianRidge # KernelRidge # ExtraTreesRegressor # XgBoost models = [LinearRegression(),Ridge(),Lasso(alpha=0.01,max_iter=10000),RandomForestRegressor(),GradientBoostingRegressor(),SVR(),LinearSVR(), ElasticNet(alpha=0.001,max_iter=10000),SGDRegressor(max_iter=1000,tol=1e-3),BayesianRidge(),KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5), ExtraTreesRegressor(),XGBRegressor()] names = ["LR", "Ridge", "Lasso", "RF", "GBR", "SVR", "LinSVR", "Ela","SGD","Bay","Ker","Extra","Xgb"] # for name, model in zip(names, models): # score = rmse_cv(model, X_scaled, y_log) # print("{}: {:.6f}, {:.4f}".format(name,score.mean(),score.std())) # grid函数寻找最优参数 class grid(): def __init__(self, model): self.model = model def grid_get(self, X, y, param_grid): grid_search = GridSearchCV(self.model, param_grid, cv=5, scoring="neg_mean_squared_error") grid_search.fit(X, y)
def krr(): # KRR # linear print () tl = time.time() #alphas = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0] alphas = [1.0] lin_krr = KernelRidge(alpha=alphas, kernel='linear') hyperparams = {'alpha' : alphas} lin_krr_clf = GridSearchCV(lin_krr, hyperparams, cv=5) lin_krr_fit = lin_krr_clf.fit(cv_train_X, cv_train_Y) lin_krr_res = lin_krr_clf.cv_results_ lin_krr_params = lin_krr_clf.best_params_ lin_krr_score = lin_krr_clf.best_score_ test_pred = lin_krr_clf.predict(test_X) # The Coefficients print('Test Estimator : \n', lin_krr_clf.best_estimator_) # Mean Squared Error mse = float(m_s_e(test_Y, test_pred)) print('Test Mean Squared Error : %f' % mse) # Root Mean Squared Error rmse = float(np.sqrt(m_s_e(test_Y, test_pred))) print('Test Root Mean Squared Error : %f' % rmse) # R^2 (Coefficient of Determination) Regression Score rsq = float(r2_s(test_Y, test_pred)) print('Test R^2 Regression Score : %f' % rsq) tl = time.time() - tl print ('Time Secs : %f' % tl) # polynomial print () tp = time.time() alphas = [1.0] degs = [2, 4, 7] # M hyperparams = {'alpha' : alphas, 'degree' : degs} poly_krr = KernelRidge(kernel='poly', alpha=alphas, degree=degs, gamma=1, coef0=1) poly_krr_clf = GridSearchCV(poly_krr, hyperparams, cv=5) poly_krr_fit = poly_krr_clf.fit(cv_train_X, cv_train_Y) poly_krr_res = poly_krr_clf.cv_results_ poly_krr_params = poly_krr_clf.best_params_ poly_krr_score = poly_krr_clf.best_score_ test_pred = poly_krr_clf.predict(test_X) # The Coefficients print('Test Estimator : \n', poly_krr_clf.best_estimator_) # Mean Squared Error mse = float(m_s_e(test_Y, test_pred)) print('Test Mean Squared Error : %f' % mse) # Root Mean Squared Error rmse = float(np.sqrt(m_s_e(test_Y, test_pred))) print('Test Root Mean Squared Error : %f' % rmse) # R^2 (Coefficient of Determination) Regression Score rsq = float(r2_s(test_Y, test_pred)) print('Test R^2 Regression Score : %f' % rsq) tp = time.time() - tp print ('Time Secs : %f' % tp) # gaussian/rbf print () tg = time.time() alphas = [1.0] sigmas = [0.1, 0.5, 1.0, 2.0, 4.0] hyperparams = {'alpha' : alphas, 'gamma' : sigmas} rbf_krr = KernelRidge(kernel='rbf', alpha=alphas, gamma=sigmas) rbf_krr_clf = GridSearchCV(rbf_krr, hyperparams, cv=5) rbf_krr_fit = rbf_krr_clf.fit(cv_train_X, cv_train_Y) rbf_krr_res = rbf_krr_clf.cv_results_ rbf_krr_params = rbf_krr_clf.best_params_ rbf_krr_score = rbf_krr_clf.best_score_ test_pred = rbf_krr_clf.predict(test_X) # The Coefficients print('Test Estimator : \n', rbf_krr_clf.best_estimator_) # Mean Squared Error mse = float(m_s_e(test_Y, test_pred)) print('Test Mean Squared Error : %f' % mse) # Root Mean Squared Error rmse = float(np.sqrt(m_s_e(test_Y, test_pred))) print('Test Root Mean Squared Error : %f' % rmse) # R^2 (Coefficient of Determination) Regression Score rsq = float(r2_s(test_Y, test_pred)) print('Test R^2 Regression Score : %f' % rsq) tg = time.time() - tg print ('Time Secs : %f' % tg) return None # End of File
train_matrix1 = preprocessing.scale( train_matrix1 ) test_matrix1 = preprocessing.scale( test_matrix1 ) data2 = [ ( [ float(row[i]) for i in range(len(row) - 2) ] , ( float( row[ len(row) - 2 ] ) , float( row[ len(row) - 1 ] ) ) ) for row in csv.reader( open("default_plus_chromatic_features_1059_tracks.txt"), delimiter=',', quoting=csv.QUOTE_NONE) ] np.random.seed(0) np.random.shuffle( data2 ) train_size2 = int(len(data2) * percent) train_matrix2 = np.array( [ features for ( features, label ) in data2[0:train_size2] ] ) test_matrix2 = np.array( [ features for ( features, label ) in data2[train_size2:-1] ] ) train_labels2 = [ label for ( features , label ) in data2[0:train_size2] ] test_labels2 = [ label for ( features , label ) in data2[train_size2:-1] ] train_matrix2 = preprocessing.scale( train_matrix2 ) test_matrix2 = preprocessing.scale( test_matrix2 ) print ("") print ("Method = Linear ridge regression - Default features") model = KernelRidge( kernel='linear' ) model.fit( train_matrix1 , train_labels1 ) results = model.predict( test_matrix1 ) print ("Mean error = " + repr( np.mean( [ geodistance( results[i] , test_labels1[i] ) for i in range(results.shape[0]) ] ) ) ) print ("Median error = " + repr( np.median( [ geodistance( results[i] , test_labels1[i] ) for i in range(results.shape[0]) ] ) ) ) print ("Method = Linear ridge regression - Default features + chromatic features") model = KernelRidge( kernel='linear' ) model.fit( train_matrix2 , train_labels2 ) results = model.predict( test_matrix2 ) print ("Mean error = " + repr( np.mean( [ geodistance( results[i] , test_labels2[i] ) for i in range(results.shape[0]) ] ) ) ) print ("Median error = " + repr( np.median( [ geodistance( results[i] , test_labels2[i] ) for i in range(results.shape[0]) ] ) ) ) print ("") print ("Method = Random forest regression - Default features") model = RandomForestRegressor( n_estimators=100 , random_state=0 ) model.fit( train_matrix1 , train_labels1 )
#print(f_vector) #1*vector_size shape-like array X = np.array(X).reshape(-1, vector_size) y = np.array(y).ravel() # In[ ]: from sklearn.model_selection import train_test_split from sklearn.kernel_ridge import KernelRidge # In[ ]: X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) # In[ ]: clf = KernelRidge(alpha=1.0, kernel=my_kernel) clf.fit(X_train, y_train) # In[ ]: train_score = clf.score(X_train, y_train) test_score = clf.score(X_test, y_test) print('train score: {0}; test score: {1}'.format(train_score, test_score)) # In[ ]: from plot_learning_curve import * title = "Learning Curves (Kernel Ridge Regression)" cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0) plot_learning_curve(clf, title, X, y, cv=cv, n_jobs=4) plt.show()
diff_fano = fano[5,neuron_keep]-fano[3,neuron_keep] diff_fano = diff_fano[~np.isnan(diff_fano)] pna.cal_CohenD(diff_fano) """ temp_script """ x, y = data_tuning_mean[:,:,-10:-1].ravel(), data_tuning_std[:,:,-10:-1].ravel() kr = KernelRidge() kr.fit(x,y) kr = kernel_regression.KernelReg(y, x, ['c'], bw=[np.std(x)/5]) plt.plot(x,y, '.') plt.plot(x, kr.fit(x)[0], 'o') for i in range(data_tuning_mean.shape[0]): plot_kr(data_tuning_mean[i, :, -3].ravel(), data_tuning_std[i, :, -3].ravel(), color=colors[i], linestyle=linestyles[i]) """ legacy code """ data_neuro_cur = signal_align.select_signal(data_neuro_spk, chan_filter=range( 0,32), sortcode_filter=range(1,4)) data_neuro_cur = signal_align.select_signal(data_neuro_spk, chan_filter=range(33,48), sortcode_filter=range(1,4)) plt.figure()
#%% model = NuSVR(gamma='scale', nu=0.9, tol=0.01) oof_svr1, prediction_svr1 = train_model(params=None, model_type='sklearn', model=model) #%% params = {'loss_function': 'MAE'} oof_cat, prediction_cat = train_model(params=params, model_type='cat') #%% model = KernelRidge(kernel='rbf') oof_r, prediction_r = train_model(params=None, model_type='sklearn', model=model) #%% plt.figure(figsize=(18, 8)) plt.subplot(2, 3, 1) plt.plot(y_tr, color='g', label='y_train') plt.plot(oof_lgb, color='b', label='lgb') plt.legend(loc=(1, 0.5)) plt.title('lgb') plt.subplot(2, 3, 2) plt.plot(y_tr, color='g', label='y_train') plt.plot(oof_xgb, color='teal', label='xgb')
X = np.arange(194)[:,None] y = z X_plot = np.arange(0,210)[:,None] ############################################################################# # Fit regression model train_size = 18630 C = 3e6 gamma = 0.01 svr = SVR(kernel='rbf', C=C, gamma=gamma) alpha = 0.23 gamma1 = 0.01 kr = KernelRidge(kernel='rbf', gamma=gamma1,alpha = alpha) t0 = time.time() svr.fit(X[:train_size], y[:train_size]) svr_fit = time.time() - t0 t0 = time.time() kr.fit(X[:train_size], y[:train_size]) kr_fit = time.time() - t0 t0 = time.time() y_svr = svr.predict(X_plot) svr_predict = time.time() - t0 t0 = time.time() y_kr = kr.predict(X_plot)
print(np.isnan(train_SalePrice).any()) print(np.isnan(train_data).any()) print(np.isfinite(train_data).all()) print(np.isfinite(train_SalePrice).all()) score = rmse_cv(lasso) print("Lasso Score: ", score.mean()) score = rmse_cv(ridge) print("ridge Score: ", score.mean()) """output Lasso Score: 0.13320361773268208 ridge Score: 0.21324437336842056 """ KRR = KernelRidge() score = rmse_cv(KRR) print("KRR Score: ", score.mean()) """output KRR Score: 0.14740376549230474 """ GB = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5)
sig_h[k] = np.std(Xh_tr[:, k]) Xh_tr[:, k] = (Xh_tr[:, k] - mea_h[k]) / sig_h[k] ############## Kernel Ridge Regression ######################################## from sklearn.kernel_ridge import KernelRidge import scipy.io as sio mf = sio.loadmat( "/data/ISOTROPIC/regression/KRR_rbf_cv_alpha_gamma_sspacing4_tspacing6.mat", squeeze_me=True, struct_as_record=False ) KRR_alpha_opt = mf["KRR_alpha_opt"] print("Optimal alpha:", KRR_alpha_opt) KRR_gamma_opt = mf["KRR_gamma_opt"] print("Optimal gamma:", KRR_gamma_opt) kr = KernelRidge(kernel="rbf", alpha=KRR_alpha_opt, gamma=KRR_gamma_opt) kr.fit(Xl_tr, Xh_tr) ############## Prediction and save to file #################################### import os try: os.remove("/data/ISOTROPIC/data/KRR_rbf_sspacing4_tspacing6.nc") except OSError: pass ncfile2 = Dataset("/data/ISOTROPIC/data/KRR_rbf_sspacing4_tspacing6.nc", "w") ncfile1 = Dataset("/data/ISOTROPIC/data/data_downsampled4.nc", "r") # create the dimensions
def main(): data_train, data_test = loadData() data_train = deleteOutlier(data_train) trainX = data_train.drop(['Id', 'SalePrice'], axis=1) trainY = data_train['SalePrice'] trainY = np.log1p(trainY) ntrain = len(trainX) testId = data_test['Id'] testX = data_test.drop(['Id'], axis=1) dataSet = pd.concat([trainX, testX], axis=0) dataSet = imputingMissingData(dataSet) dataSet = transformStr(dataSet) dataSet = transformSortNum(dataSet) dataSet = extractFeat(dataSet) dataSet = boxCoxFeat(dataSet) dataSet = dummy(dataSet) trainX = dataSet[:ntrain] testX = dataSet[ntrain:] models = { # "lgd": lgb.LGBMRegressor(objective='regression', num_leaves=5, # learning_rate=0.05, n_estimators=720, # max_bin=55, bagging_fraction=0.8, # bagging_freq=5, feature_fraction=0.2319, # feature_fraction_seed=9, bagging_seed=9, # min_data_in_leaf=6, min_sum_hessian_in_leaf=11), 'gbdt': GradientBoostingRegressor(n_estimators=2000, learning_rate=0.01, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=12, loss='huber', random_state=0), "ridge": Ridge(alpha=0.05, max_iter=100), 'krr': KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5), 'enet': ElasticNet(alpha=0.0005, l1_ratio=1, random_state=3), 'lasso': Lasso(alpha=0.0008, max_iter=100, random_state=1), } stackReg = StackingRegressor( regressors=[models['gbdt'], models['krr'], models['enet']], meta_regressor=models['lasso']) modelStack1 = StackingAverageModels(base_models=(models['ridge'], models['krr'], models['enet']), meta_model=models['lasso']) modelStack2 = StackingAverageModels( base_models=(models['gbdt'], models['ridge'], models['krr'], models['lasso']), meta_model=models['enet']) modelStack3 = StackingAverageModels(base_models=(models['gbdt'], models['lasso'], models['enet']), meta_model=models['krr']) modelStack4 = StackingAverageModels(base_models=(models['lasso'], models['krr'], models['enet']), meta_model=models['gbdt']) averageModel = AveragingModels( (modelStack1, modelStack2, modelStack3, modelStack4)) # for key in models: # scores, reg = train(trainX, trainY, models[key]) # # print('model: {} scores: {}'.format(key, scores)) # parameters = {"n_estimators": [500], "min_samples_split": [12, 15], "min_samples_leaf": [12, 15], # "max_depth": [4, 6], "random_state": [0]} # clf = GridSearchCV(models['gbdt'], parameters) scores, reg = train(trainX.values, trainY.values, modelStack2) test(testX.values, testId, reg) print(scores)
##### regr = linear_model.LinearRegression() scores = cross_val_score(regr, data.df[inputVariables].values, data.df['count'].values) print("Linear Regression cross validation score: ", scores.mean()) regr.fit(X_train_sum, y_train_sum) print("Linear Regression training score: ", regr.score(X_train_sum, y_train_sum)) print("Linear Regression testing score: ", regr.score(X_test_sum, y_test_sum)) ##### Kernel Ridge and Support Vector Regression ##### ## Finding the best parameters alpha=[1,1e-1,1e-2,1e-3] for a in alpha: kr = KernelRidge(kernel='rbf', alpha=a) kr.fit(X_train_sum, y_train_sum) print("Kernel Ridge train score: ", kr.score(X_train_sum, y_train_sum), " for alpha = %s" %a) print("Kernel Ridge test score: ", kr.score(X_test_sum, y_test_sum), " for alpha = %s" %a) ### Using GridSearchCV param_grid = { 'alpha': [1, 1e-1, 1e-2] "gamma": np.logspace(-2, 2, 5) } GSKernelRidge = GridSearchCV(KernelRidge(kernel='rbf'), param_grid=param_grid) GSKernelRidge.fit(X_train_sum, y_train_sum)
import numpy as np from sklearn.kernel_ridge import KernelRidge import matplotlib.pyplot as plt from sklearn.model_selection import GridSearchCV # rng = np.random.RandomState(0) x = 5 * np.random.rand(100, 1) #构成的形状为(100,1) y = np.sin(x).ravel() #add noise to target y[::5] += 3 * (0.5 - np.random.rand(x.shape[0] // 5)) # kr = KernelRidge(kernel='sigmoid',gamma=0.4) kr = GridSearchCV(KernelRidge(), param_grid={ "kernel": ["rbf", "laplacian", "polynomial", "sigmoid"], "alpha": [1e0, 0.1, 1e-2, 1e-3], "gamma": np.logspace(-2, 2, 5) }) kr.fit(x, y) print(kr.best_score_, kr.best_params_) x_plot = np.linspace(0, 5, 100) y_kr = kr.predict(x_plot[:, None]) plt.scatter(x, y) plt.plot(x_plot, y_kr, "r") plt.show()
X = X_orig[:, :-n_removed] Y = wine['target'] # scale evrything X_scaled = preprocessing.scale(X) X_orig_scaled = preprocessing.scale(X_orig) # train test split X_train, X_test, Y_train, Y_test = train_test_split(X_scaled, Y, test_size=0.2, random_state=5) # let us train the model kernel = RBF(length_scale=10) krr_model = KernelRidge(kernel=kernel, alpha=1) krr_model.fit(X_train, Y_train) # and restrict to the first coordinates def my_model(array): return krr_model.predict(array[:, :-n_removed]) # dimension of the ambient space dim = X_orig.shape[1] # the example to explain xi = X_orig_scaled[0, :] # bandwidth parameter nu = 5
import matplotlib.pyplot as plt from sklearn.linear_model import Ridge from sklearn.linear_model import LinearRegression from sklearn.svm import SVR from sklearn.kernel_ridge import KernelRidge ridg = Ridge(alpha=0.01) #svr = SVR(kernel='poly') svr = GridSearchCV(SVR(kernel='linear', gamma=0.1), cv=5, param_grid={ "C": [1e0, 1e1, 1e2, 1e3], "gamma": np.logspace(-2, 2, 5) }) kr = GridSearchCV(KernelRidge(kernel='linear', gamma=0.1), cv=5, param_grid={ "alpha": [1e0, 0.1, 1e-2, 1e-3], "gamma": np.logspace(-2, 2, 5) }) def get_coef(x, y, model=ridg): #x=np.array([-10.1, -8.9, 0, 5.2, 10.1]).reshape(-1, 1) #y=np.array([-19, -18.1, 2, 10, 21]).reshape(-1, 1) if model == ridg: y = np.array(y, dtype=float).reshape(-1, 1) x = np.array(x, dtype=float).reshape(-1, 1) else:
rng = np.random.RandomState(0) # Generate sample data X = 15 * rng.rand(100, 1) y = np.sin(X).ravel() y += 3 * (0.5 - rng.rand(X.shape[0])) # add noise # Fit KernelRidge with parameter selection based on 5-fold cross validation param_grid = { "alpha": [1e0, 1e-1, 1e-2, 1e-3], "kernel": [ ExpSineSquared(l, p) for l in np.logspace(-2, 2, 10) for p in np.logspace(0, 2, 10) ] } kr = GridSearchCV(KernelRidge(), param_grid=param_grid) stime = time.time() kr.fit(X, y) print("Time for KRR fitting: %.3f" % (time.time() - stime)) gp_kernel = ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) \ + WhiteKernel(1e-1) gpr = GaussianProcessRegressor(kernel=gp_kernel) stime = time.time() gpr.fit(X, y) print("Time for GPR fitting: %.3f" % (time.time() - stime)) # Predict using kernel ridge X_plot = np.linspace(0, 20, 10000)[:, None] stime = time.time() y_kr = kr.predict(X_plot)
cutarr2 = cut_predict['x'].values cutarr2 = cutarr2.astype(np.float64) for x in np.nditer(cutarr2, op_flags=['readwrite']): x[...] -= 733000 cutarr2 = cutarr2.reshape(-1, 1) outarr = new_cut['y'].values outarr = outarr.astype(np.float64) outarr = outarr.reshape(-1, 1) outarr2 = cut_predict['y'].values outarr2 = outarr2.astype(np.float64) outarr2 = outarr2.reshape(-1, 1) regressor = KernelRidge(alpha=0.005, kernel='rbf', gamma=0.00025, degree=3, coef0=1, kernel_params=None) regressor2 = SVR(C=1e3, epsilon=0.01, gamma=0.001, tol=1e-4) regressor.fit(cutarr, outarr) time1, time2 = [], [] for i in range( dt.date(2008, 2, 25).toordinal(), dt.date(2014, 2, 3).toordinal(), 7): time1 = np.append(time1, i) for i in range( dt.date(2014, 10, 27).toordinal(), dt.date(2018, 8, 21).toordinal(), 7): time2 = np.append(time2, i)
for row in csv.DictReader(open("affective-ratings.csv")): affective[ row["Word"].lower() ] = np.array( [ float( row["V.Mean.Sum"] ) , float( row["A.Mean.Sum"] ) , float( row["D.Mean.Sum"] ) ] ) # Expand dictionary of affective words embeddings_dim = 300 max_words = 100000 embeddings = dict( ) embeddings = Word2Vec.load_word2vec_format( "GoogleNews-vectors-negative300.bin.gz" , binary=True ) train_matrix = [ ] train_labels = [ ] for word,scores in affective.items(): try: train_matrix.append( embeddings[word] ) train_labels.append( scores ) except: continue model = KernelRidge( kernel='poly' , degree=4 ) model.fit( train_matrix , train_labels ) textdata = " ".join( open(sys.argv[1] + ".revised.txt",'r').readlines( ) ) tokenizer = Tokenizer(nb_words=max_words, filters=keras.preprocessing.text.base_filter(), lower=True, split=" ") tokenizer.fit_on_texts( textdata ) for word, index in tokenizer.word_index.items(): try: if not affective.has_key(word) : affective[word] = np.array( model.predict( np.array( embedding[word] ).reshape(1, -1) )[0] ) except: affective[word] = np.array( [ 5.0 , 5.0 , 5.0 ] ) # Process the textual contents textdata = "" file1 = open(sys.argv[1] + ".revised.txt",'r') with file1 as myfile: textdata = re.sub( ">", ">" , re.sub("<" , "<" , re.sub( "&" , "&" , re.sub( " +", "\n\n" , re.sub( "\t" , " ", re.sub( "\r" , "" , "".join( myfile.readlines() ) ) ) ) ) ) ) corenlp = StanfordCoreNLP( ) file2 = open(sys.argv[1] + ".annotated.tsv",'w')
# Set the parameters by cross-validation tuned_parameters = { 'kernel': ['rbf'], 'coef0': [1e-4, 1e-3, 1e-2, 1e2, 1e3], 'gamma': [1e-4, 1e-3, 1e-2, 1e2, 1e3], 'alpha': [1e-3, 1e-2, 1e-1, 0, 1e1, 1e2, 1e3] } scores = ['neg_mean_absolute_error'] model = 'KRR' for score in scores: # print(kernel[i]) print("# Tuning hyper-parameters for %s" % score) print() if args.random == True: clf = RandomizedSearchCV(KernelRidge(), tuned_parameters, cv=5, verbose=10, n_jobs=-1, scoring='%s' % score) start_rn = time.time() clf.fit(X_train, y_train) end_rn = time.time() if args.grid == True: clf = GridSearchCV(KernelRidge(), tuned_parameters, cv=5, verbose=10, n_jobs=-1,
if __name__=="__main__": #trains Kronecker RLS for different sample sizes #comparing CPU time and verifying that the learned #dual coefficients are same for both methods regparam = 1.0 for size in [10, 20, 40, 60, 80, 100, 500, 1000, 2000, 4000, 6000]: X1, X2, y = random_data(size, 100) kernel1 = GaussianKernel(X1, gamma=0.01) K1 = kernel1.getKM(X1) kernel2 = GaussianKernel(X2, gamma=0.01) K2 = kernel2.getKM(X2) start = time.clock() rls = KronRLS(K1=K1, K2=K2, Y=y, regparam=regparam) dur = time.clock() - start print("RLScore pairs: %d, CPU time: %f" %(size**2, dur)) #forming full Kronecker product kernel matrix becomes fast #unfeasible if size <=100: K = np.kron(K2, K1) start = time.clock() ridge = KernelRidge(alpha=regparam, kernel="precomputed") ridge.fit(K, y) dur = time.clock() - start print("sklearn pairs: %d, CPU time: %f" %(size**2, dur)) sklearn_coef = ridge.dual_coef_ core_coef = rls.predictor.A.reshape(K1.shape[0], K2.shape[0]).T.ravel() print("Are the coefficients same: %r" %np.allclose(sklearn_coef, core_coef)) else: print("sklearn: too much data") print "*****"
def fit(self): """Train the model with the indicated algorithm. Do not forget to tune the hyperparameters. Parameters ---------- algorithm : String, "KernelRidge", "SVM", "LinearRegression", "Lasso", "ElasticNet", "NeuralNet", "BaggingNeuralNet", default = "SVM" """ self.X_scaler.fit(self.X_train) self.Y_scaler.fit(self.y_train) # scaling the data in all cases, it may not be used during the fit later self.X_train_sc = self.X_scaler.transform(self.X_train) self.y_train_sc = self.Y_scaler.transform(self.y_train) self.X_test_sc = self.X_scaler.transform(self.X_test) self.y_test_sc = self.Y_scaler.transform(self.y_test) if self.algorithm == "KernelRidge": clf_kr = KernelRidge(kernel=self.user_kernel) self.model = sklearn.model_selection.GridSearchCV( clf_kr, cv=5, param_grid=self.param_kr) elif self.algorithm == "SVM": clf_svm = SVR(kernel=self.user_kernel) self.model = sklearn.model_selection.GridSearchCV( clf_svm, cv=5, param_grid=self.param_svm) elif self.algorithm == "Lasso": clf_lasso = sklearn.linear_model.Lasso( alpha=0.1, random_state=self.rand_state) self.model = sklearn.model_selection.GridSearchCV( clf_lasso, cv=5, param_grid=dict(alpha=np.logspace(-5, 5, 30))) elif self.algorithm == "ElasticNet": clf_ElasticNet = sklearn.linear_model.ElasticNet( alpha=0.1, l1_ratio=0.5, random_state=self.rand_state) self.model = sklearn.model_selection.GridSearchCV( clf_ElasticNet, cv=5, param_grid=dict(alpha=np.logspace(-5, 5, 30))) elif self.algorithm == "LinearRegression": self.model = sklearn.linear_model.LinearRegression() elif self.algorithm == "NeuralNet": self.model = MLPRegressor(**self.param_neurons) elif self.algorithm == "BaggingNeuralNet": nn_m = MLPRegressor(**self.param_neurons) self.model = BaggingRegressor(base_estimator=nn_m, **self.param_bag) if self.scaling == True: self.model.fit(self.X_train_sc, self.y_train_sc.reshape(-1, )) predict_train_sc = self.model.predict(self.X_train_sc) self.prediction_train = self.Y_scaler.inverse_transform( predict_train_sc.reshape(-1, 1)) predict_test_sc = self.model.predict(self.X_test_sc) self.prediction_test = self.Y_scaler.inverse_transform( predict_test_sc.reshape(-1, 1)) else: self.model.fit(self.X_train, self.y_train.reshape(-1, )) self.prediction_train = self.model.predict(self.X_train) self.prediction_test = self.model.predict(self.X_test)
def parametrize_environment_specific(settings, rerun): channel_name = settings["embedding_options"]["channel_name"] log << log.mg << "Parametrizing" << channel_name << "model" << log.endl soap_types = SETTINGS["soap_types"] log << "Particle SOAP types are" << ", ".join(soap_types) << log.endl # PATHS - for example: # { "xyz_file": "data_esol/structures.xyz", # "soap_file": "data_esol/structures.soap", # "kmat_file": "data_esol/kernel.npy", # "targets_file": "data_esol/targets.npy", # "range_file": "data_esol/range.json", # "weights_file": "data_esol/weights.npy" } paths = copy.deepcopy(settings["paths"]) for p,v in paths.iteritems(): paths[p] = os.path.join(PATH, v) log << "Path to %s = %s" % (p, paths[p]) << log.endl configs = soap.tools.io.read(paths["xyz_file"]) # SOAP soap_options = SETTINGS["soap_options"][settings["soap_options_ref"]] if rerun or not os.path.isfile(paths["soap_file"]): log << "Make target: %s" % paths["soap_file"] << log.endl soap_configure_default(types=soap_types) dset = soap_evaluate(configs, soap_options, paths["soap_file"]) else: log << "Load target: %s" % paths["soap_file"] << log.endl dset = soap.DMapMatrixSet(paths["soap_file"]) # KERNEL kernel_options = settings["kernel_options"] if rerun or not os.path.isfile(paths["kmat_file"]): log << "Make target: %s" % paths["kmat_file"] << log.endl K = kernel_evaluate(dset, kernel_options, paths["kmat_file"]) else: log << "Load target: %s" % paths["kmat_file"] << log.endl K = np.load(paths["kmat_file"]) # TARGETS target_key = settings["regression_options"]["target_key"] if rerun or not os.path.isfile(paths["targets_file"]): log << "Make target: %s" % paths["targets_file"] << log.endl targets = np.array([float(c.info[target_key]) for c in configs]) np.save(paths["targets_file"], targets) else: log << "Load target: %s" % paths["targets_file"] << log.endl targets = np.load(paths["targets_file"]) # MODEL regr_options = settings["regression_options"] if rerun or not os.path.isfile(paths["weights_file"]): log << "Make target: %s" % paths["weights_file"] << log.endl y_avg = np.average(targets) krr = KernelRidge( alpha=regr_options["lreg"], kernel='precomputed') krr.fit(K**regr_options["xi"], targets) y_predict = krr.predict(K**regr_options["xi"]) kweights = krr.dual_coef_ np.save(paths["weights_file"], kweights) np.save(paths["pred_file"], y_predict) else: log << "Load target: %s" % paths["weights_file"] << log.endl kweights = np.load(paths["weights_file"]) y_predict = np.load(paths["pred_file"]) if rerun or not os.path.isfile(paths["range_file"]): dset_attr = soap.DMapMatrixSet(paths["soap_file"]) delta_Ys = kernel_attribute(dset_attr, dset, kernel_options, kweights, regr_options["xi"]) json.dump(delta_Ys, open(paths["range_file"], "w")) else: delta_Ys = json.load(open(paths["range_file"]))
rawData = np.zeros((4104, 4)) with open('Processed Time Dependent Data.csv', 'rb') as f: reader = csv.reader(f) labels = reader.next() i = 0 for row in reader: for j in range(4): rawData[i, j] = float(row[j]) i = i + 1 kr = KernelRidge(alpha=1, kernel='rbf', gamma=None, degree=3, coef0=1, kernel_params=None) kr.fit(rawData[:, 0:3], rawData[:, 3]) samplePoints = np.zeros((6000, 4)) with open('Sample Points 2.csv', 'rb') as dat: readDat = csv.reader(dat) i = 0 for row in readDat: for j in range(3): samplePoints[i, j] = float(row[j]) i = i + 1 for iterno in range(1, 13):
class RidgeMKL: """A MKL model in a transductive setting (test points are presented at training time). """ mkls = { "align": Align, "alignf": Alignf, "alignfc": Alignf, "uniform": UniformAlignment, } mkls_low_rank = { "align": AlignLowRank, "alignf": AlignfLowRank, "alignfc": AlignfLowRank, "uniform": UniformAlignmentLowRank, } # alignf expects kernels to be centered centered = {"alignf", "alignfc"} supervised = {"align", "alignf", "alignfc"} def __init__(self, lbd=0, method="align", method_init_args={}, low_rank=False): """ :param method: (``string``) "align", "alignf", or "uniform", MKL method to be used. :param low_rank: (``bool``) Use low-rank approximations. :param method_init_args: (``dict``) Initialization arguments for the MKL methods. :param lbd: (``float``) L2-regularization. """ self.method = method if not low_rank: self.mkl_model = self.mkls[method](**method_init_args) if method == "alignfc": init_args = method_init_args.copy() init_args["typ"] = "convex" self.mkl_model = self.mkls[method](**init_args) else: self.mkl_model = self.mkls_low_rank[method](**method_init_args) if method == "alignfc": init_args = method_init_args.copy() init_args["typ"] = "convex" self.mkl_model = self.mkls_low_rank[method](**init_args) self.lbd = lbd self.low_rank = low_rank self.trained = False def fit(self, Ks, y, holdout=None): """Learn weights for kernel matrices or Kinterfaces. :param Ks: (``list``) of (``numpy.ndarray``) or of (``Kinterface``) to be aligned. :param y: (``numpy.ndarray``) Class labels :math:`y_i \in {-1, 1}` or regression targets. :param holdout: (``list``) List of indices to exlude from alignment. """ # Expand kernel interfaces to kernel matrices expand = lambda K: K[:, :] if isinstance(K, Kinterface) else K Hs = map(expand, Ks) # Assert correct dimensions assert Ks[0].shape[0] == len(y) # Fit MKL model if self.method in self.supervised: self.mkl_model.fit(Hs, y, holdout=holdout) else: self.mkl_model.fit(Hs) if self.low_rank: self.X = hstack(map(lambda e: sqrt(e[0]) * e[1], zip(self.mkl_model.mu, Hs))) if self.method in self.centered: self.X = center_kernel_low_rank(self.X) self.X[where(isnan(self.X))] = 0 # Fit ridge model with given lbd and MKL model self.ridge = KernelRidge(alpha=self.lbd, kernel="linear", ) # Fit ridge on the examples minus the holdout set inxs = list(set(range(Hs[0].shape[0])) - set(holdout)) self.ridge.fit(self.X[inxs], y[inxs]) self.trained = True else: # Fit ridge model with given lbd and MKL model self.ridge = KernelRidge(alpha=self.lbd, kernel=self.mkl_model, ) # Fit ridge on the examples minus the holdout set inxs = array(list(set(range(Hs[0].shape[0])) - set(holdout))) inxs = inxs.reshape((len(inxs), 1)).astype(int) self.ridge.fit(inxs, y[inxs]) self.trained = True def predict(self, inxs): """ Predict values for data on indices inxs (transcductive setting). :param inxs: (``list``) Indices of samples to be used for prediction. :return: (``numpy.ndarray``) Vector of prediction of regression targets. """ assert self.trained if self.low_rank: return self.ridge.predict(self.X[inxs]) else: inxs = array(inxs) inxs = inxs.reshape((len(inxs), 1)).astype(int) return self.ridge.predict(inxs).ravel()
def make_CV_models(X, y): '''performs grid searches to find all the best models for dataset X, y''' model_dict = { 'KRR': grid_search(X, y, KernelRidge(), param_grid={ "alpha": np.logspace(-10, 2, 300), "gamma": np.logspace(-10, -1, 100), "kernel": ['rbf'] }), 'SVR': grid_search(X, y, SVR(), param_grid={ "C": np.logspace(-1, 4, 20), "epsilon": np.logspace(-2, 2, 20) }), 'Ridge': grid_search(X, y, Ridge(), param_grid={"alpha": np.logspace(-6, 6, 150)}), 'Lasso': grid_search(X, y, Lasso(max_iter=20000), param_grid={"alpha": np.logspace(-2, 6, 100)}), 'BR': grid_search(X, y, BayesianRidge(), param_grid={ "alpha_1": np.logspace(-13, -5, 10), "alpha_2": np.logspace(-9, -3, 10), "lambda_1": np.logspace(-10, -5, 10), "lambda_2": np.logspace(-11, -4, 10) }), 'GBoost': grid_search(X, y, GradientBoostingRegressor(), param_grid={ "n_estimators": np.linspace(5, 350, 100).astype('int') }), 'RF': grid_search( X, y, RandomForestRegressor(), param_grid={"n_estimators": np.linspace(5, 100, 50).astype('int')}, ), 'kNN': grid_search( X, y, KNeighborsRegressor(), param_grid={"n_neighbors": np.linspace(2, 20, 18).astype('int')}), 'mean': DummyRegressor(strategy='mean'), } return model_dict
px = [] py = [] with open('/home/redwards/Desktop/genus_species_analysis/pseudo_coverage.txt', 'r') as fin: for l in fin: p = l.strip().split("\t") px.append(float(p[0])) py.append(float(p[1])) ny = np.array(y) nx = np.array(x) pnx = np.array(px) pny = np.array(py) kr = KernelRidge(kernel='rbf', gamma=7.5e-5, alpha=0.001) kr.fit(nx[:, None], ny[:, None]) x_pred = np.linspace(min(x), max(x), 10000)[:, None] y_pred = kr.predict(x_pred) kr.fit(pnx[:, None], pny[:, None]) px_pred = np.linspace(min(px), max(px), 10000)[:, None] py_pred = kr.predict(px_pred) fig = plt.figure() ax = fig.add_subplot(111) """
def ml_size_scan(x_datafile, y_datafile, ids, testfiles, alpha0=1, gamma0=1, kernel0='rbf', learning_rate_init0=0.001, hidden_layer_sizes0=(80, 80, 80), sample_size_list=[ 0.005, 0.01, 0.03, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9 ], ml_method="krr"): max_sample_size = max(sample_size_list) ax_train, x_test, ay_train, y_test, aids_train, ids_test = load_split_scale_data( x_datafile, y_datafile, ids, testfiles, max_sample_size) for sample_size in sample_size_list: ratio = float(sample_size) / float(max_sample_size) if ratio > 0.999: x_train = ax_train y_train = ay_train ids_train = aids_train else: # reduce set x_dump, x_train, y_dump, y_train, ids_dump, ids_train = train_test_split( ax_train, ay_train, aids_train, test_size=ratio, ) if ml_method == "krr": # Create kernel linear ridge regression object learner = KernelRidge(alpha=alpha0, coef0=1, degree=3, gamma=gamma0, kernel=kernel0, kernel_params=None) elif ml_method == "mlp": # Create Multi-Layer Perceptron object learner = MLPRegressor(hidden_layer_sizes=hidden_layer_sizes0, max_iter=1600, alpha=alpha0, learning_rate_init=learning_rate_init0) else: print("ML method unknown. Exiting.") exit(1) t_ml0 = time.time() learner.fit(x_train, y_train) t_ml1 = time.time() print("ml time", str(t_ml1 - t_ml0)) mae, mse, y_pred, train_y_pred, learner = predict_and_error( learner, x_test, x_train, y_test) ### OUTPUT ### write_output(learner, sample_size, ml_method, mae, mse, "size", ids_test, y_test, y_pred, ids_train, y_train, train_y_pred) return None
##################################################################### # --- RUN THE MODEL: FOR A GIVEN SPLIT AND EACH PARAMETER TRIAL --- # ##################################################################### # For each parameter trial for i in xrange(trials): # For regression use the Kernel Ridge method if model_type == "regression": print "\n Starting experiment for trial %d and parameter alpha = %3f\n " % (i, alpha_grid[i]) # Fit the kernel ridge model KR = KernelRidge(kernel = 'precomputed', alpha = alpha_grid[i]) KR.fit(K_train, y_train) # predict on the validation and test set y_pred = KR.predict(K_val) y_pred_test = KR.predict(K_test) # adjust prediction: needed because the training targets have been normalizaed y_pred = y_pred * float(y_train_std) + y_train_mean y_pred_test = y_pred_test * float(y_train_std) + y_train_mean # root mean squared error on validation rmse = np.sqrt(mean_squared_error(y_val, y_pred)) perf_all_val.append(rmse) # root mean squared error in test
def create(X, X_column_types, y, y_column_types, arm, **kwargs): method = kwargs.get("method", "RFE_rf") method = kwargs.get("method", "RFE_Lasso") method = kwargs.get("method", "Lasso") method = kwargs.get("method", "Boruta") method = kwargs.get("method", "Autoencoder") method = kwargs.get("method", "Boruta") method = kwargs.get("method", "XBG") # finding distance correlation b/w the features def distcorr(X, Y): X = np.atleast_1d(X) Y = np.atleast_1d(Y) if np.prod(X.shape) == len(X): X = X[:, None] if np.prod(Y.shape) == len(Y): Y = Y[:, None] X = np.atleast_2d(X) Y = np.atleast_2d(Y) n = X.shape[0] if Y.shape[0] != X.shape[0]: raise ValueError('Number of samples must match') a = squareform(pdist(X)) b = squareform(pdist(Y)) A = a - a.mean(axis=0)[None, :] - a.mean(axis=1)[:, None] + a.mean() B = b - b.mean(axis=0)[None, :] - b.mean(axis=1)[:, None] + b.mean() dcov2_xy = (A * B).sum() / float(n * n) dcov2_xx = (A * A).sum() / float(n * n) dcov2_yy = (B * B).sum() / float(n * n) dcor = np.sqrt(dcov2_xy) / np.sqrt( np.sqrt(dcov2_xx) * np.sqrt(dcov2_yy)) return dcor ############# Shuffle the data ################### X['target'] = y data = shuffle(X) X = data.drop(['target'], axis=1) y = data['target'] # AutoLearn will take only the numerical features num_X = X.select_dtypes(include=[int, float]) #################### preprocessing step ################### if (y_column_types == "int64" or y_column_types == "float64"): # Regression #if method == "mutual_information": c, d = num_X.shape names = np.arange(d) mode = SelectKBest(mutual_info_regression, k='all') mode.fit_transform(num_X, y) feats = sorted(zip(map(lambda c: round(c, 4), mode.scores_), names), reverse=True) finale = [] for i in range(0, len(feats)): r, s = feats[i] if (r > 0): finale.append(s) dataframe = num_X.iloc[:, finale] else: # classification c, d = num_X.shape names = np.arange(d) mode = SelectKBest(mutual_info_classif, k='all') mode.fit_transform(num_X, y) feats = sorted(zip(map(lambda c: round(c, 4), mode.scores_), names), reverse=True) finale = [] for i in range(0, len(feats)): r, s = feats[i] if (r > 0): finale.append(s) dataframe = num_X.iloc[:, finale] ############# feature_generation ######################### Non_linear = pd.DataFrame() linear = pd.DataFrame() m, n = dataframe.shape thrs = 0.7 for i in range(n): for j in range(i + 1, n): if (i != j) and (distcorr(dataframe.iloc[:, i], dataframe.iloc[:, j]) != 0): if (distcorr(dataframe.iloc[:, i], dataframe.iloc[:, j]) > 0) and (distcorr(dataframe.iloc[:, i], dataframe.iloc[:, j]) < thrs): non_lin_X, non_lin_y = dataframe.iloc[:, i].to_frame( ), dataframe.iloc[:, j] model = KernelRidge(alpha=1.0, coef0=1, degree=3, gamma=None, kernel='rbf', kernel_params=None) model.fit(non_lin_X, non_lin_y) first = model.predict(non_lin_X) first_feat_gen = pd.Series(first) second_feat_gen = (non_lin_y - (first)) Non_linear = Non_linear.append(first_feat_gen, ignore_index=True) Non_linear = Non_linear.append(second_feat_gen, ignore_index=True) elif (distcorr(dataframe.iloc[:, i], dataframe.iloc[:, j]) >= thrs) and (distcorr(dataframe.iloc[:, i], dataframe.iloc[:, j]) <= 1): lin_X, lin_y = dataframe.iloc[:, i].to_frame( ), dataframe.iloc[:, j] model = Ridge(alpha=1.0) model.fit(lin_X, lin_y) first_feat = model.predict(lin_X) lin_first_feat = pd.Series(first_feat) second_feat = (lin_y - (first_feat)) linear = linear.append(lin_first_feat, ignore_index=True) linear = linear.append(second_feat, ignore_index=True) nonlinear_genereted = Non_linear.T linear_generated = linear.T print("no. of features generated by nonlinear features :", nonlinear_genereted.shape[1]) print("no. of features generated by linear features:", linear_generated.shape[1]) Generated_feats = pd.concat([nonlinear_genereted, linear_generated], axis=1) print("Total no. of generated features :", Generated_feats.shape[1]) #################### Feature_selection in 2 steps################### if Generated_feats.shape[1] > 0: ############ 1st step of selction ############## if method == "RFE_rf": if (y_column_types == "int64" or y_column_types == "float64"): model = RandomForestRegressor() else: model = RandomForestClassifier() rfe = RFECV(model, step=1, cv=5) rfe.fit(Generated_feats, y) print("Optimal number of features after 1st step : %d" % rfe.n_features_) selected_feats_order = np.argsort(rfe.grid_scores_)[::-1] Data_X = pd.DataFrame() for i in range(rfe.n_features_): col = Generated_feats.iloc[:, selected_feats_order[i]] Data_X = Data_X.append(col) one_featsel = Data_X.transpose() elif method == "RFE_Lasso": if (y_column_types == "int64" or y_column_types == "float"): model = LassoLarsCV() else: model = LogisticRegressionCV(penalty="l1", solver='liblinear') rfe = RFECV(model, step=1, cv=5) rfe.fit(Generated_feats, y) print("Optimal number of features after 1st step : %d" % rfe.n_features_) selected_feats_order = np.argsort(rfe.grid_scores_)[::-1] Data_X = pd.DataFrame() for i in range(rfe.n_features_): col = Generated_feats.iloc[:, selected_feats_order[i]] Data_X = Data_X.append(col) one_featsel = Data_X.transpose() elif method == "Lasso": if (y_column_types == "int64" or y_column_types == "float64"): model = LassoLarsCV(eps=1e-8) else: model = LogisticRegressionCV(penalty="l1", solver='liblinear') sfm = SelectFromModel(model, threshold=0.7) sfm.fit(Generated_feats, y) n_features = sfm.transform(Generated_feats).shape[1] print("optimal no. of features after lst step:", n_features) one_featsel = pd.DataFrame(sfm.transform(Generated_feats)) elif method == "Boruta": if (y_column_types == "int64" or y_column_types == "float"): model = XGBRegressor() else: model = XGBClassifier() boruta = BorutaPy(model, n_estimators='auto', verbose=2) boruta.fit(Generated_feats.values, y.values) sel_index = Generated_feats.columns[boruta.support_] one_featsel = Generated_feats.loc[:, sel_index] elif method == "XGB": if (y_column_types == "int64" or y_column_types == "float64"): model = XGBRegressor() else: model = XGBClassifier() sfm = SelectFromModel(model, threshold=0.7) sfm.fit(Generated_feats, y) n_features = sfm.transform(Generated_feats).shape[1] print("optimal no. of features after lst step:", n_features) one_featsel = pd.DataFrame(sfm.transform(Generated_feats)) #elif method == "Autoencoder" : ################# 2nd step of slections ########### o, p = one_featsel.shape Names = np.arange(p) if (y_column_types == "int64" or y_column_types == "float"): model = SelectKBest(mutual_info_regression, k='all') else: model = SelectKBest(mutual_info_classif, k='all') model.fit_transform(one_featsel, y) new_feat = sorted(zip(map(lambda o: round(o, 4), model.scores_), Names), reverse=True) sel_finale = [] for i in range(0, len(new_feat)): s, t = new_feat[i] if (s > 0): sel_finale.append(t) second_featsel = one_featsel.iloc[:, sel_finale] ################ concat the original features & selected new features transformed_X = pd.concat([X, second_featsel], axis=1) else: transformed_X = X return None, transformed_X
# Choose the number of predicted peptides and their length n_predictions = 1000 y_length = 5 # Max time (seconds) for the branch and bound search max_time = 500 print('String maximization model on BPPs dataset') gs_kernel = GenericStringKernel(AminoAcidFile.blosum62_natural, sigma_position, sigma_amino_acid, n, is_normalized=True) alphabet = gs_kernel.alphabet dataset = load_bpps_dataset() # Use a regression algorithm to learn the weights first print('Learning the regression weights ...') learner = KernelRidge(alpha, kernel='precomputed') gram_matrix = gs_kernel(dataset.X, dataset.X) learner.fit(gram_matrix, dataset.y) learned_weights = learner.dual_coef_ # We can then use the string maximization model with the learned weights print('Branch and bound search for the top {} peptides of length {} ...'.format(n_predictions, y_length)) model = StringMaximizationModel(alphabet, n, gs_kernel, max_time) model.fit(dataset.X, learned_weights, y_length) peptides, bioactivities = model.predict(n_predictions) print('\n') print('Peptides | Predicted bioactivities') for peptide, bioactivity in zip(peptides, bioactivities): print(peptide, bioactivity)
def train(self): final = KernelRidge(alpha=.1, kernel="linear") self.csvToArray() final.fit(self.feats, self.labels) self.model = final
class VADEstimator(BaseEstimator): def fit( self, x , y , size=1 ): self.model = Sequential() self.model.add(Dense( int( embeddings_dim / 2.0 ) , input_dim=embeddings_dim , init='uniform' , activation='tanh')) self.model.add(Dense( int( embeddings_dim / 4.0 ) , init='uniform' , activation='tanh')) self.model.add(Dense(size , init='uniform' ) ) self.model.compile(loss='mse', optimizer='rmsprop') self.model = KernelRidge( kernel='rbf' ) self.model.fit( x , y ) def predict( self, x ): if isinstance( self.model , Sequential ): return self.model.predict( x , verbose=0 )[ 0 ] return self.model.predict( x )
def test_generalization_across_time(): """Test time generalization decoding """ from sklearn.svm import SVC from sklearn.base import is_classifier # KernelRidge is used for testing 1) regression analyses 2) n-dimensional # predictions. from sklearn.kernel_ridge import KernelRidge from sklearn.preprocessing import LabelEncoder from sklearn.metrics import roc_auc_score, mean_squared_error epochs = make_epochs() y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1)) if check_version('sklearn', '0.18'): from sklearn.model_selection import (KFold, StratifiedKFold, ShuffleSplit, LeaveOneLabelOut) cv_shuffle = ShuffleSplit() cv = LeaveOneLabelOut() # XXX we cannot pass any other parameters than X and y to cv.split # so we have to build it before hand cv_lolo = [(train, test) for train, test in cv.split( X=y_4classes, y=y_4classes, labels=y_4classes)] # With sklearn >= 0.17, `clf` can be identified as a regressor, and # the scoring metrics can therefore be automatically assigned. scorer_regress = None else: from sklearn.cross_validation import (KFold, StratifiedKFold, ShuffleSplit, LeaveOneLabelOut) cv_shuffle = ShuffleSplit(len(epochs)) cv_lolo = LeaveOneLabelOut(y_4classes) # With sklearn < 0.17, `clf` cannot be identified as a regressor, and # therefore the scoring metrics cannot be automatically assigned. scorer_regress = mean_squared_error # Test default running gat = GeneralizationAcrossTime(picks='foo') assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat) assert_raises(ValueError, gat.fit, epochs) with warnings.catch_warnings(record=True): # check classic fit + check manual picks gat.picks = [0] gat.fit(epochs) # check optional y as array gat.picks = None gat.fit(epochs, y=epochs.events[:, 2]) # check optional y as list gat.fit(epochs, y=epochs.events[:, 2].tolist()) assert_equal(len(gat.picks_), len(gat.ch_names), 1) assert_equal( "<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no " "prediction, no score>", '%s' % gat) assert_equal(gat.ch_names, epochs.ch_names) # test different predict function: gat = GeneralizationAcrossTime(predict_method='decision_function') gat.fit(epochs) # With classifier, the default cv is StratifiedKFold assert_true(gat.cv_.__class__ == StratifiedKFold) gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1)) gat.predict_method = 'predict_proba' gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 2)) gat.predict_method = 'foo' assert_raises(NotImplementedError, gat.predict, epochs) gat.predict_method = 'predict' gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1)) assert_equal( "<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), " "predicted 14 epochs, no score>", "%s" % gat) gat.score(epochs) assert_true(gat.scorer_.__name__ == 'accuracy_score') # check clf / predict_method combinations for which the scoring metrics # cannot be inferred. gat.scorer = None gat.predict_method = 'decision_function' assert_raises(ValueError, gat.score, epochs) # Check specifying y manually gat.predict_method = 'predict' gat.score(epochs, y=epochs.events[:, 2]) gat.score(epochs, y=epochs.events[:, 2].tolist()) assert_equal( "<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), " "predicted 14 epochs,\n scored " "(accuracy_score)>", "%s" % gat) with warnings.catch_warnings(record=True): gat.fit(epochs, y=epochs.events[:, 2]) old_mode = gat.predict_mode gat.predict_mode = 'super-foo-mode' assert_raises(ValueError, gat.predict, epochs) gat.predict_mode = old_mode gat.score(epochs, y=epochs.events[:, 2]) assert_true("accuracy_score" in '%s' % gat.scorer_) epochs2 = epochs.copy() # check _DecodingTime class assert_equal( "<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: " "0.050 (s), length: 0.050 (s), n_time_windows: 15>", "%s" % gat.train_times_) assert_equal( "<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: " "0.050 (s), length: 0.050 (s), n_time_windows: 15 x 15>", "%s" % gat.test_times_) # the y-check gat.predict_mode = 'mean-prediction' epochs2.events[:, 2] += 10 gat_ = copy.deepcopy(gat) with use_log_level('error'): assert_raises(ValueError, gat_.score, epochs2) gat.predict_mode = 'cross-validation' # Test basics # --- number of trials assert_true(gat.y_train_.shape[0] == gat.y_true_.shape[0] == len( gat.y_pred_[0][0]) == 14) # --- number of folds assert_true(np.shape(gat.estimators_)[1] == gat.cv) # --- length training size assert_true( len(gat.train_times_['slices']) == 15 == np.shape(gat.estimators_)[0]) # --- length testing sizes assert_true( len(gat.test_times_['slices']) == 15 == np.shape(gat.scores_)[0]) assert_true( len(gat.test_times_['slices'][0]) == 15 == np.shape(gat.scores_)[1]) # Test score_mode gat.score_mode = 'foo' assert_raises(ValueError, gat.score, epochs) gat.score_mode = 'fold-wise' scores = gat.score(epochs) assert_array_equal(np.shape(scores), [15, 15, 5]) gat.score_mode = 'mean-sample-wise' scores = gat.score(epochs) assert_array_equal(np.shape(scores), [15, 15]) gat.score_mode = 'mean-fold-wise' scores = gat.score(epochs) assert_array_equal(np.shape(scores), [15, 15]) gat.predict_mode = 'mean-prediction' with warnings.catch_warnings(record=True) as w: gat.score(epochs) assert_true( any("score_mode changed from " in str(ww.message) for ww in w)) # Test longer time window gat = GeneralizationAcrossTime(train_times={'length': .100}) with warnings.catch_warnings(record=True): gat2 = gat.fit(epochs) assert_true(gat is gat2) # return self assert_true(hasattr(gat2, 'cv_')) assert_true(gat2.cv_ != gat.cv) with warnings.catch_warnings(record=True): # not vectorizing scores = gat.score(epochs) assert_true(isinstance(scores, np.ndarray)) # type check assert_equal(len(scores[0]), len(scores)) # shape check assert_equal(len(gat.test_times_['slices'][0][0]), 2) # Decim training steps gat = GeneralizationAcrossTime(train_times={'step': .100}) with warnings.catch_warnings(record=True): gat.fit(epochs) gat.score(epochs) assert_true(len(gat.scores_) == len(gat.estimators_) == 8) # training time assert_equal(len(gat.scores_[0]), 15) # testing time # Test start stop training & test cv without n_fold params y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1)) train_times = dict(start=0.090, stop=0.250) gat = GeneralizationAcrossTime(cv=cv_lolo, train_times=train_times) # predict without fit assert_raises(RuntimeError, gat.predict, epochs) with warnings.catch_warnings(record=True): gat.fit(epochs, y=y_4classes) gat.score(epochs) assert_equal(len(gat.scores_), 4) assert_equal(gat.train_times_['times'][0], epochs.times[6]) assert_equal(gat.train_times_['times'][-1], epochs.times[9]) # Test score without passing epochs & Test diagonal decoding gat = GeneralizationAcrossTime(test_times='diagonal') with warnings.catch_warnings(record=True): # not vectorizing gat.fit(epochs) assert_raises(RuntimeError, gat.score) with warnings.catch_warnings(record=True): # not vectorizing gat.predict(epochs) scores = gat.score() assert_true(scores is gat.scores_) assert_equal(np.shape(gat.scores_), (15, 1)) assert_array_equal( [tim for ttime in gat.test_times_['times'] for tim in ttime], gat.train_times_['times']) # Test generalization across conditions gat = GeneralizationAcrossTime(predict_mode='mean-prediction', cv=2) with warnings.catch_warnings(record=True): gat.fit(epochs[0:6]) with warnings.catch_warnings(record=True): # There are some empty test folds because of n_trials gat.predict(epochs[7:]) gat.score(epochs[7:]) # Test training time parameters gat_ = copy.deepcopy(gat) # --- start stop outside time range gat_.train_times = dict(start=-999.) with use_log_level('error'): assert_raises(ValueError, gat_.fit, epochs) gat_.train_times = dict(start=999.) assert_raises(ValueError, gat_.fit, epochs) # --- impossible slices gat_.train_times = dict(step=.000001) assert_raises(ValueError, gat_.fit, epochs) gat_.train_times = dict(length=.000001) assert_raises(ValueError, gat_.fit, epochs) gat_.train_times = dict(length=999.) assert_raises(ValueError, gat_.fit, epochs) # Test testing time parameters # --- outside time range gat.test_times = dict(start=-999.) with warnings.catch_warnings(record=True): # no epochs in fold assert_raises(ValueError, gat.predict, epochs) gat.test_times = dict(start=999.) with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat.predict, epochs) # --- impossible slices gat.test_times = dict(step=.000001) with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat.predict, epochs) gat_ = copy.deepcopy(gat) gat_.train_times_['length'] = .000001 gat_.test_times = dict(length=.000001) with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat_.predict, epochs) # --- test time region of interest gat.test_times = dict(step=.150) with warnings.catch_warnings(record=True): # not vectorizing gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 5, 14, 1)) # --- silly value gat.test_times = 'foo' with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat.predict, epochs) assert_raises(RuntimeError, gat.score) # --- unmatched length between training and testing time gat.test_times = dict(length=.150) assert_raises(ValueError, gat.predict, epochs) # --- irregular length training and testing times # 2 estimators, the first one is trained on two successive time samples # whereas the second one is trained on a single time sample. train_times = dict(slices=[[0, 1], [1]]) # The first estimator is tested once, the second estimator is tested on # two successive time samples. test_times = dict(slices=[[[0, 1]], [[0], [1]]]) gat = GeneralizationAcrossTime(train_times=train_times, test_times=test_times) gat.fit(epochs) with warnings.catch_warnings(record=True): # not vectorizing gat.score(epochs) assert_array_equal(np.shape(gat.y_pred_[0]), [1, len(epochs), 1]) assert_array_equal(np.shape(gat.y_pred_[1]), [2, len(epochs), 1]) # check cannot Automatically infer testing times for adhoc training times gat.test_times = None assert_raises(ValueError, gat.predict, epochs) svc = SVC(C=1, kernel='linear', probability=True) gat = GeneralizationAcrossTime(clf=svc, predict_mode='mean-prediction') with warnings.catch_warnings(record=True): gat.fit(epochs) # sklearn needs it: c.f. # https://github.com/scikit-learn/scikit-learn/issues/2723 # and http://bit.ly/1u7t8UT with use_log_level('error'): assert_raises(ValueError, gat.score, epochs2) gat.score(epochs) assert_true(0.0 <= np.min(scores) <= 1.0) assert_true(0.0 <= np.max(scores) <= 1.0) # Test that gets error if train on one dataset, test on another, and don't # specify appropriate cv: gat = GeneralizationAcrossTime(cv=cv_shuffle) gat.fit(epochs) with warnings.catch_warnings(record=True): gat.fit(epochs) gat.predict(epochs) assert_raises(ValueError, gat.predict, epochs[:10]) # Make CV with some empty train and test folds: # --- empty test fold(s) should warn when gat.predict() gat._cv_splits[0] = [gat._cv_splits[0][0], np.empty(0)] with warnings.catch_warnings(record=True) as w: gat.predict(epochs) assert_true(len(w) > 0) assert_true( any('do not have any test epochs' in str(ww.message) for ww in w)) # --- empty train fold(s) should raise when gat.fit() gat = GeneralizationAcrossTime(cv=[([0], [1]), ([], [0])]) assert_raises(ValueError, gat.fit, epochs[:2]) # Check that still works with classifier that output y_pred with # shape = (n_trials, 1) instead of (n_trials,) if check_version('sklearn', '0.17'): # no is_regressor before v0.17 gat = GeneralizationAcrossTime(clf=KernelRidge(), cv=2) epochs.crop(None, epochs.times[2]) gat.fit(epochs) # With regression the default cv is KFold and not StratifiedKFold assert_true(gat.cv_.__class__ == KFold) gat.score(epochs) # with regression the default scoring metrics is mean squared error assert_true(gat.scorer_.__name__ == 'mean_squared_error') # Test combinations of complex scenarios # 2 or more distinct classes n_classes = [2, 4] # 4 tested # nicely ordered labels or not le = LabelEncoder() y = le.fit_transform(epochs.events[:, 2]) y[len(y) // 2:] += 2 ys = (y, y + 1000) # Univariate and multivariate prediction svc = SVC(C=1, kernel='linear', probability=True) reg = KernelRidge() def scorer_proba(y_true, y_pred): return roc_auc_score(y_true, y_pred[:, 0]) # We re testing 3 scenario: default, classifier + predict_proba, regressor scorers = [None, scorer_proba, scorer_regress] predict_methods = [None, 'predict_proba', None] clfs = [svc, svc, reg] # Test all combinations for clf, predict_method, scorer in zip(clfs, predict_methods, scorers): for y in ys: for n_class in n_classes: for predict_mode in ['cross-validation', 'mean-prediction']: # Cannot use AUC for n_class > 2 if (predict_method == 'predict_proba' and n_class != 2): continue y_ = y % n_class with warnings.catch_warnings(record=True): gat = GeneralizationAcrossTime( cv=2, clf=clf, scorer=scorer, predict_mode=predict_mode) gat.fit(epochs, y=y_) gat.score(epochs, y=y_) # Check that scorer is correctly defined manually and # automatically. scorer_name = gat.scorer_.__name__ if scorer is None: if is_classifier(clf): assert_equal(scorer_name, 'accuracy_score') else: assert_equal(scorer_name, 'mean_squared_error') else: assert_equal(scorer_name, scorer.__name__)
f5 = open("f5.csv") data5 = np.loadtxt(f5, delimiter=',') dataTemp5 = data5[:, np.newaxis] index5 = list(np.array(dataTemp5).reshape(-1,)) index5 = [int(i-1) for i in index5] #Combine the index index = [index1, index2, index3, index4, index5] index = np.asarray(index) #Model: a = .00139;y = .518 model = KernelRidge(alpha= a, gamma=y, kernel='rbf') #Loop: rmse_list = [] num_folds = 5 #data is divide into 5 time slices Overall_Y_Pred = np.zeros(len(X)) for i in [t+1 for t in list(range(4))]: to_exclude = list(range(i)) folder_train = np.asarray(to_exclude).astype(int) #index_train starts with the first folder index_train = index[folder_train]; index_test = [element for i, element in enumerate(index) if i not in to_exclude] print (len(index_test)) #train set starts with the first folder
from sklearn.linear_model import Ridge from sklearn.datasets import load_diabetes, load_boston from sklearn.preprocessing import MinMaxScaler # %% # constants num_splits = 9 num_seeds = 10 DATASET_LOADER = {"boston": load_boston, "diabetes": load_diabetes} MODELS = { "linear_ridge": Ridge(alpha=5.0), "kernel_ridge": KernelRidge(kernel="rbf"), # "linear_svr": LinearSVR(max_iter=12000), "svr": SVR(kernel="rbf", gamma="scale", C=1.0, epsilon=0.1) } # %% def line(x, y, *args, **kwargs): # a = np.minimum(x.min(), y.min()) # b = np.maximum(x.max(), y.max()) a = 0.2 b = 2.0 u = [a, b]
class Learner(): path = 'matrices/' inputF = 'inputs.npy' stateF = 'states.npy' itrF = 'itr.npy' inptFile = os.path.join(path, inputF) stateFile = os.path.join(path, stateF) itrFile = os.path.join(path, itrF) itr = np.array([]) useSHIV = False THRESH = 0.45 ahqp_solver_g = AHQP(sigma=6) ahqp_solver_b = AHQP(sigma=5,nu=1e-3) def trainModel(self, s=None, a=None): """ Trains model on given states and actions. Uses neural net or SVM based on global settings. """ states, actions = self.states[3:], self.actions[3:] #print "states.shape" #print states.shape #print "actions.shape" #print actions.shape if len(self.itr) == 0: self.itr = np.array([states.shape[0]]) else: self.itr = np.hstack((self.itr, states.shape[0])) '''if states.shape[0] > 2700.0: f = os.path.join(self.path, 'statesToValidate.npy') np.save(f, states) IPython.embed()''' fits = [] #actions = actions.ravel() self.clf = KernelRidge(alpha=1.0) self.clf.kernel = 'rbf' print "SIZE: ", states.shape self.clf.fit(states, actions) #IPython.embed() actions_pred = self.clf.predict(states) bad_state = np.zeros(actions_pred.shape[0]) for i in range(actions_pred.shape[0]): fit = LA.norm(actions_pred[i,:] - actions[i,:]) fits.append(fit) med = np.median(np.array(fits)) for fit in fits: if(fit>med): bad_state[i] = 1 IPython.embed() if self.useSHIV: self.labels = np.zeros(states.shape[0])+1.0 self.scaler = preprocessing.StandardScaler().fit(states) states_proc = self.scaler.transform(states) good_labels = bad_state == 0.0 states_g = states_proc[good_labels,:] bad_labels = bad_state == 1.0 states_b = states_proc[bad_labels,:] #IPython.embed() self.ahqp_solver_g.assembleKernel(states_g, np.zeros(states_g.shape[0])+1.0) self.ahqp_solver_b.assembleKernel(states_b, np.zeros(states_b.shape[0])+1.0) #IPython.embed() self.ahqp_solver_g.solveQP() self.ahqp_solver_b.solveQP() #score = self.clf.score(states, actions) #print score self.plot(fits, states, med) def askForHelp(self,state): if self.useSHIV: state = self.scaler.transform(state) if self.ahqp_solver_b.predict(state)==1.0: return -1.0 else: return self.ahqp_solver_g.predict(state) else: return -1 def plot(self, fits, states, threshold): index = range(len(states)) t = np.ones(len(index)) * threshold plt.figure(1) plt.plot(index, fits, color='b', linewidth=4.0) plt.plot(index, t, color='r', linewidth=4.0) plt.ylabel('Fit') plt.xlabel('Index of State') plt.show() def getAction(self, state): """ Returns a prediction given the input state. Uses neural net or SVM based on global settings. """ return self.clf.predict(state) def initModel(self, useSHIV): self.useSHIV = useSHIV try: self.states = np.load(self.stateFile) self.actions = np.load(self.inptFile) except IOError: self.states = np.array([-8,8.75,0,-12,22,0,-15,21.13043404, 0,-12,18.52173996,0,-15,14.173913, 0,-12,8.08695698,0,0,0,0,0]) self.actions = np.array([0,0,0,0]) #self.trainModel(self.states, self.actions) def updateModel(self, s, a): self.states = np.vstack((self.states, s)) self.actions = np.vstack((self.actions,a)) #self.trainModel(self.states, self.actions) def saveModel(self): path = 'matrices/oldData/' currT = strftime("%Y-%m-%d %H:%M:%S", gmtime()) inptFileOut = os.path.join(path, 'inputs' + currT + '.npy') stateFileOut = os.path.join(path, 'states' + currT + '.npy') np.save(stateFileOut, self.states) np.save(inptFileOut, self.actions) np.save(self.itrFile, self.itr)
y = np.sin(X).ravel() # Add noise to targets y[::5] += 3 * (0.5 - rng.rand(X.shape[0] // 5)) X_plot = np.linspace(0, 5, 100000)[:, None] # ############################################################################# # Fit regression model train_size = 100 svr = GridSearchCV(SVR(kernel='rbf', gamma=0.1), param_grid={"C": [1e0, 1e1, 1e2, 1e3], "gamma": np.logspace(-2, 2, 5)}, n_jobs=1) kr = GridSearchCV(KernelRidge(kernel='rbf', gamma=0.1), param_grid={"alpha": [1e0, 0.1, 1e-2, 1e-3], "gamma": np.logspace(-2, 2, 5)}, n_jobs=1) t0 = time.time() svr.fit(X[:train_size], y[:train_size]) svr_fit = time.time() - t0 print("SVR complexity and bandwidth selected and model fitted in %.3f s" % svr_fit) t0 = time.time() kr.fit(X[:train_size], y[:train_size]) kr_fit = time.time() - t0 print("KRR complexity and bandwidth selected and model fitted in %.3f s" % kr_fit)
df = df.apply(LabelEncoder().fit_transform) X = df.drop('Face Rental', axis=1).values y = df['Face Rental'].values X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1, test_size=0.3) regressor = [ SVR(kernel='rbf', gamma=0.7, C=1), linear_model.Ridge(alpha=.5), linear_model.Lasso(alpha=0.1), linear_model.LassoLars(alpha=.1), linear_model.BayesianRidge(), MLPRegressor(), DecisionTreeRegressor(), KernelRidge(), PassiveAggressiveRegressor(), RANSACRegressor(), TheilSenRegressor(), RandomForestRegressor() ] result_cols = ["Regressor", "Accuracy"] result_frame = pd.DataFrame(columns=result_cols) for model in regressor: name = model.__class__.__name__ model.fit(X_train, y_train) predictions = model.predict(X_test) error = sqrt(mean_squared_error(y_test, predictions)) acc = 100 - error
def ml_param_size_scan(x_datafile, y_datafile, ids, testfiles, alpha_list=np.logspace(-1, -8, 8), gamma_list=np.logspace(-2, -10, 9), kernel_list=['rbf'], layer_list=[(40, 40, 40)], learning_rate_list=[0.001], sample_size_list=[ 0.005, 0.01, 0.03, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9 ], ml_method="krr", paramset_size=0.1): print('model ' + ml_method) max_sample_size = max(sample_size_list) # load, split and scale data ax_train, ax_test, ay_train, ay_test, aids_train, aids_test = load_split_scale_data( x_datafile, y_datafile, ids, testfiles, max_sample_size) # search for optimal learner parameters # reduce set x_train, x_test, y_train, y_test, ids_train, ids_test = train_test_split( ax_train, ay_train, aids_train, test_size=1 - paramset_size, ) if ml_method == "krr": # Create kernel linear ridge regression object learner = GridSearchCV(KernelRidge(kernel='rbf'), n_jobs=8, cv=5, param_grid={ "alpha": alpha_list, "gamma": gamma_list, "kernel": kernel_list }, scoring='neg_mean_absolute_error') elif ml_method == "mlp": # Create Multi-Layer Perceptron object learner = GridSearchCV(MLPRegressor(hidden_layer_sizes=(40, 40, 40), max_iter=1600, alpha=0.001, learning_rate_init=0.001), n_jobs=8, cv=5, param_grid={ "alpha": alpha_list, "learning_rate_init": learning_rate_list, "hidden_layer_sizes": layer_list }, scoring='neg_mean_absolute_error') else: print("ML method unknown. Exiting.") exit(1) t_ml0 = time.time() learner.fit(x_train, y_train) t_ml1 = time.time() print("ml time", str(t_ml1 - t_ml0)) # getting best parameters learner_best = learner.best_estimator_ mae, mse, y_pred, train_y_pred, learner_best = predict_and_error( learner_best, x_test, x_train, y_test) ### OUTPUT ### write_output(learner, max_sample_size * paramset_size, ml_method, mae, mse, "param", ids_test, y_test, y_pred, ids_train, y_train, train_y_pred) # use above found best parameters x_test = ax_test y_test = ay_test ids_test = aids_test paramlearner = learner for sample_size in sample_size_list: ratio = float(sample_size) / float(max_sample_size) if ratio > 0.999: x_train = ax_train y_train = ay_train ids_train = aids_train else: # reduce set ("test set" is the part of the training set that is used) x_dump, x_train, y_dump, y_train, ids_dump, ids_train = train_test_split( ax_train, ay_train, aids_train, test_size=ratio, ) if ml_method == "krr": # Create kernel linear ridge regression object alpha0 = paramlearner.best_params_["alpha"] gamma0 = paramlearner.best_params_["gamma"] kernel0 = paramlearner.best_params_["kernel"] learner = KernelRidge(alpha=alpha0, coef0=1, degree=3, gamma=gamma0, kernel=kernel0, kernel_params=None) elif ml_method == "mlp": # Create Multi-Layer Perceptron object hidden_layer_sizes0 = paramlearner.best_params_[ "hidden_layer_sizes"] alpha0 = paramlearner.best_params_["alpha"] learning_rate_init0 = paramlearner.best_params_[ "learning_rate_init"] learner = MLPRegressor(hidden_layer_sizes=hidden_layer_sizes0, max_iter=1600, alpha=alpha0, learning_rate_init=learning_rate_init0) else: print("ML method unknown. Exiting.") exit(1) t_ml0 = time.time() learner.fit(x_train, y_train) t_ml1 = time.time() print("ml time", str(t_ml1 - t_ml0)) mae, mse, y_pred, train_y_pred, learner = predict_and_error( learner, x_test, x_train, y_test) ### OUTPUT ### write_output(learner, sample_size, ml_method, mae, mse, "psize", ids_test, y_test, y_pred, ids_train, y_train, train_y_pred) return None