def train(filename,X_train, Y_train, Y_Label): nn = MLPRegressor(hidden_layer_sizes= (100,100,100)) X = np.array(X_train[0:1000]) Y = np.array(Y_train[0:1000]) nn.fit(X, Y) test = nn.predict(np.array(X_train[1000:])) # plot predict data and real data train = np.array(Y_train[1000:]) x = np.arange(0,len(X_train)-1000) plt.plot(x,train) plt.plot(x,test) plt.show() print (Y_train[1000:]) train_pred = nn.score(np.array(X_train[0:1000]), np.array(Y_train[0:1000]), sample_weight=None) test_pred = nn.score(np.array(X_train[1000:]), np.array(Y_train[1000:]), sample_weight=None) print test_pred # predict label using trained regression # label_pred = test.tolist() # for i in range(len(label_pred)-1, 0,-1): # if label_pred[i]-label_pred[i-1] > 0: label_pred[i] = 1 # else: label_pred[i] = 0 # label_pred[0] = 0 # count = 0 # for (i, j) in zip(label_pred,Y_Label[1000:]): # if label_pred[i] == Y_Label[i]: count+=1 # label_score.append(count*1.0/len(label_pred)) pickle.dump(nn, open(filename, 'wb')) print (nn.get_params())
def fit(self, verbose=False):
data = self.data
# split into train and test data
train = data.T[0:150000]
test = data.T[150000:]
x_train = train.T[0:6].T
y_train = train.T[6].T
x_test = test.T[0:6].T
y_test = test.T[6].T
# train the model
mlp = MLPRegressor()
mlp.hidden_layer_sizes = self.hidden_layer_sizes
mlp.activation = self.activation
mlp.solver = self.solver
mlp.alpha = self.alpha
mlp.fit(x_train, y_train)
self.mlp = mlp
# predict
y_train_p = mlp.predict(x_train)
y_test_p = mlp.predict(x_test)
# print training and test error
train_error = np.mean((y_train - y_train_p)**2)
test_error = np.mean((y_test - y_test_p)**2)
if verbose:
print(mlp.get_params())
print("average training error is : %f" % (train_error))
print("average test error is : %f" % (test_error))
return train_error, test_error
def NN_Skit(x_train, y_train, x_test):
from sklearn.neural_network import MLPRegressor
m = MLPRegressor(hidden_layer_sizes=(100,100,100),alpha=0.001,solver='adam',activation='relu',max_iter=1,random_state=1,verbose=True)
#m = MLPRegressor(hidden_layer_sizes=(100,100,100,100,),random_state=1)
#m = MLPRegressor(random_state=1)
m.fit(x_train,y_train)
print("Neural Network Score: {}".format(m.score(x_train,y_train)))
print("Hyperparameters: {}".format(m.get_params()))
mean_train = m.predict(x_train)
mean_test = m.predict(x_test)
return mean_train, mean_test, m
def train(filename, a, b, X_train, Y_train,Y_Label): m = len(X_train)*a/b # X_train = norm(X_train) nn = MLPRegressor() # shuffle(zip(X_train, Y_train)) X = np.array(X_train[0:m]) Y = np.array(Y_train[0:m]) nn.fit(X, Y) test = nn.predict(np.array(X_train[m:])) # plot predict data and real data train = np.array(Y_train[m:]) x = np.arange(0,len(Y_train[m:])) plt.plot(x,train) plt.plot(x,test) plt.show() print nn.score(np.array(X_train[0:m]), np.array(Y_train[0:m]), sample_weight=None) print nn.score(np.array(X_train[m:]), np.array(Y_train[m:]), sample_weight=None) pickle.dump(nn, open(filename, 'wb')) print (nn.get_params()) X_class = [] Y_class = [] for i in range(0, len(X_train)): if Y_Label[i] == -1: continue X_class.append(X_train[i]) Y_class.append(Y_Label[i]) m = len(X_class)*a/b X_class = norm(X_class) nn1 = MLPClassifier(hidden_layer_sizes=(50,50,50,),max_iter=20000, learning_rate_init=0.02) nn2 = SVC() X = np.array(X_class[0:m]) Y = np.array(Y_class[0:m]) nn1.fit(X, Y) nn2.fit(X, Y) print nn1.predict(np.array(X_class[:m])) print nn2.predict(np.array(X_class[:m])) print (Y_class[:m]) print nn1.score(np.array(X_class[0:m]), np.array(Y_class[0:m]), sample_weight=None) print nn1.score(np.array(X_class[m:]), np.array(Y_class[m:]), sample_weight=None) print nn2.score(np.array(X_class[0:m]), np.array(Y_class[0:m]), sample_weight=None) print nn2.score(np.array(X_class[m:]), np.array(Y_class[m:]), sample_weight=None) y_score = nn2.fit(np.array(X_class[0:m]), np.array(Y_class[0:m])).decision_function(X_class[m:]) fpr = dict() tpr = dict() roc_auc = dict() fpr, tpr, _ = roc_curve(np.array(Y_class[m:]), y_score) roc_auc = auc(fpr, tpr)
def run_experiment(hidden_layer_sizes, number_months=12, learning_rate=.001):
"""
:param hidden_layer_sizes: The hidden layers, e.g. (40, 10)
:return:
"""
mlp_regressor = MLPRegressor(
hidden_layer_sizes=hidden_layer_sizes,
activation='relu', # most likely linear effects
solver='adam', # good choice for large data sets
alpha=0.0001, # L2 penalty (regularization term) parameter.
batch_size='auto',
learning_rate_init=learning_rate,
max_iter=200,
shuffle=True,
random_state=None,
tol=0.0001,
#verbose=True,
verbose=False,
warm_start=False, # erase previous solution
early_stopping=False, # stop if no increase during validation
validation_fraction=0.1, # belongs to early_stopping
beta_1=0.9, # solver=adam
beta_2=0.999, # solver=adam
epsilon=1e-08 # solver=adam
)
setup_logger(hidden_layer_sizes, learning_rate)
logging.info("hidden_layer_sizes=%s" % str(hidden_layer_sizes))
logging.info("number_months=%i" % number_months)
logging.info("learning_rate=%f" % learning_rate)
for month in range(1, number_months):
month_learned = "2016-%02i" % month
logging.info("learn month %s" % month_learned)
train(mlp_regressor, month_learned, month_learned, verbose=(month == 1))
month_not_yet_learned = "2016-%02i" % (month + 1)
logging.info("validate with month %s" % month_not_yet_learned)
evaluate(mlp_regressor, month_not_yet_learned, month_not_yet_learned)
logging.info(mlp_regressor.get_params())
logger = logging.getLogger()
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
def main(argv):
# read in vehicles csv
vehicles_df = pd.read_csv("../data/vehicles.csv", low_memory=False)
vehicles_displ_mpg_all = vehicles_df[['displ', 'UHighway']]
vehicles_displ_mpg = vehicles_displ_mpg_all[vehicles_displ_mpg_all.displ > 0]
half = int(len (vehicles_displ_mpg) / 2)
# create the training set with the first half of data
first_half = vehicles_displ_mpg [:half]
second_half = vehicles_displ_mpg [half:]
first_half_sorted = first_half.sort_values(by=['displ', 'UHighway'])
first_half_grouped_by_mean = pd.DataFrame({'train_mean' : \
first_half_sorted.groupby('displ')['UHighway'].mean()}).reset_index()
first_half_x = first_half_grouped_by_mean ['displ'].values.reshape(-1,1)
print(type(first_half_x))
print(first_half_x.shape)
first_half_y = first_half_grouped_by_mean ['train_mean'].values.reshape(-1,1)
print(first_half_y.shape)
#ax = first_half_grouped_by_median.plot (x = "displ", y = "train_median", c = "b")
#plt.show ()
second_half_sorted = second_half.sort_values(by=['displ', 'UHighway'])
second_half_grouped_by_mean = pd.DataFrame({'test_mean' : \
second_half_sorted.groupby('displ')['UHighway'].mean()}).reset_index()
second_half_x = second_half_grouped_by_mean ['displ'].values.reshape(-1,1)
second_half_y = second_half_grouped_by_mean ['test_mean'].values.reshape(-1,1)
#second_half_grouped_by_median.plot (ax=ax, x = "displ", y = "test_median", c = "gold")
#plt.show ()
# Create linear regression object
solvers = ['lbfgs', 'sgd', 'adam']
for solver in solvers:
regr = MLPRegressor(hidden_layer_sizes=(100,), max_iter=200, alpha=1e-4,
solver=solver, verbose=10, tol=1e-4, random_state=1,learning_rate_init=.1)
# Train the model using the training sets
regr.fit(first_half_x, first_half_y.ravel())
# Make predictions using the testing set
second_half_y_pred = regr.predict(second_half_x)
print("\tModel parameters: ", regr.get_params(deep=False))
# mean squared error
mse = mean_squared_error(second_half_y, second_half_y_pred)
rmse = np.sqrt(mse)
print("Mean squared error: %.2f" % mse)
print("Root mean squared error: %.2f" % rmse)
# Explained variance score: 1 is perfect prediction
print('R-squared score: %.2f' % r2_score(second_half_y, second_half_y_pred))
# Plot outputs
plt.scatter(first_half_x, first_half_y, color='green', label = "training set")
plt.scatter(second_half_x, second_half_y, color='red', label = "testing set")
plt.plot(second_half_x, second_half_y_pred, color='blue', linewidth=3)
plt.legend(loc='upper right')
plt.xlabel ("Engine displacement (liter)")
plt.ylabel ("Fuel economy (MPG)")
plt.text(2, 60, r'$solver=%s$' %solver, fontsize=10, fontweight='bold')
plt.show()
from sklearn.neural_network import MLPRegressor # model = MLPRegressor(hidden_layer_sizes=(6,1),activation ='identity',solver = 'lbfgs', learning_rate = 'constant', # learning_rate_init =0.2,max_iter = 2000, random_state =13) model = MLPRegressor(random_state=13) # In[47]: #To get pre-training Weights and Output Values model.partial_fit(x_train, y_train) # In[48]: #Get model parameters Params = model.get_params(deep=True) type(Params) print(Params) # In[49]: #Get Inital Parameters to compare pre-training and post-training InitialWeightsI_H = model.coefs_[0] # print(InitialWeightsI_H) print(np.shape(InitialWeightsI_H)) InitialWeightsH_O = model.coefs_[1] # print(InitialWeightsH_O) print(np.shape(InitialWeightsH_O))
scaler = StandardScaler()
# Fit only to the training data
scaler.fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
X[columns] = scaler.transform(X[columns])
from sklearn.neural_network import MLPRegressor
mlp = MLPRegressor(hidden_layer_sizes=(30,30,30))
mlp.fit(X_train,y_train)
predictions = mlp.predict(X_test)
mlp.score(X_test, y_test)
mlp.get_params()
from sklearn.metrics import r2_score, explained_variance_score
print(r2_score(y_test,predictions))
print(explained_variance_score(y_test, predictions))
import seaborn as sns
sns.lmplot("x", "y", data=pd.DataFrame({"x":y_test, "y" : predictions}))
preds = mlp.predict(X)
data['preds'] = preds
data['expected_diff'] = [i-j for i,j in zip(data.price.tolist(), data.preds.tolist())]
def main():
# pull in the hyperparameters for the neural network
parser = argparse.ArgumentParser()
parser.add_argument('config_file')
parser.add_argument('--retrain',help='retrain model',action='store_true')
parser.add_argument('--val_only',help='generate validation output only',action='store_true')
parser.add_argument('--train_fullres',help='train on full resolution spectra',action='store_true')
parser.add_argument('--disable_wp',help='disable weight propagation',action='store_true')
args = parser.parse_args()
retrain = args.retrain
use_wp = (not args.disable_wp)
val_only = args.val_only
full_res = args.train_fullres
configname = pathsplit(args.config_file)[1].replace('.json','')
config = json_load_ascii(args.config_file)
wavelengthfile = config["wavelength_file"]
names = config['inputvector_order']
trainfile = config['trainfile']
trainfile_rs = config.get('trainfile_rs',None)
if 'alb' not in names:
names.append('alb')
# assume outdir is always the same as trainfile (not trainfile_rs)
outdir,trainf = pathsplit(trainfile)
wl_inst, fwhm_inst = load_wavelength_file(wavelengthfile)
# prm20151026t173213_libRadtran.mat (full res) shape = (9072, 7101)
# prm20151026t173213_libRadtran_PRISM_rs.mat (instrument) shape = (9072, 246)
if full_res:
# Train on full-resolution RTM channels, rather than (downsampled)
# instrument channels
print('loading trainfile: "%s"'%str((trainfile)))
D = loadmat(trainfile)
wl = D['wl'].squeeze()
assert(len(wl)>len(wl_inst))
else:
# Load the resampled instrument channels
print('loading trainfile_rs: "%s"'%str((trainfile_rs)))
D = loadmat(trainfile_rs)
wl = D['wl'].squeeze()
assert(len(wl)==len(wl_inst))
assert((wl==wl_inst).all())
inputs = s.float32(D['input'])
print('inputs.shape: "%s"'%str((inputs.shape)))
tgts = D['rho']
# dimensionality of state space = n_inputs
n_inputs = len(names)
print('names: "%s"'%str((names)))
assert(n_inputs == inputs.shape[1])
n_samples = len(inputs)
n_wl = len(wl)
# construct train/test partitions
random.seed(42)
samp_idx = s.arange(n_samples)
tr_mask = np.zeros(n_samples,dtype=np.bool8)
stratify_inputs=True
if stratify_inputs:
uniq = []
trinputs,teinputs = [],[]
for i in range(n_inputs):
# hold out central unique value of each variable, train on rest
holdi = np.unique(inputs[:,i])
teinputs.append(np.array([holdi[len(holdi)//2]]))
trinputs.append(np.setdiff1d(holdi,teinputs[-1]))
print('i,trinputs[i],teinputs[i]: "%s"'%str((i,trinputs[i],
teinputs[i])))
uniq.append(holdi)
# NOTE: leave all albedo values in training set
# partition on remaining (4) states
for i in range(n_inputs):
if not names[i].startswith('alb'):
tr_mask |= np.isin(inputs[:,i],teinputs[i])
# invert mask to get training indices
tr_mask = ~tr_mask
else:
# validate on (100*p)% of the data
p = 0.2
random.shuffle(samp_idx)
tr_mask[:int(n_samples*(1-p))] = 1
tr_idx = samp_idx[tr_mask]
val_idx = samp_idx[~tr_mask]
n_samplesv = len(val_idx)
print('n_samples: %d'%(n_samples))
print('n_train: %d'%(len(tr_idx)))
print('n_val: %d'%(n_samplesv))
print('n_inputs: %d'%(n_inputs))
print('n_wl: %d'%(n_wl))
# initialize model
# n_layers = n_hidden_layers + 1 (output)
n_layers = 2
n_hidden = 55
if n_layers==2:
weight_labs = ['input','output']
layers = (n_hidden,)
else:
weight_labs = ['input','hidden','output']
layers = (n_hidden,n_hidden,)
# halfwidth of overlapping channel input range
# n_half == 0 -> monochromatic
# n_half == 1->3 channel averaging, 2->5 channel averaging, ...
n_half = 0
average_over = True # only valid if n_half > 0
n_over = 2*n_half+1
# 'auto'== 200 samples/batch
batch_size = 'auto'
# train first subnetwork for many more epochs to ensure convergence
init_max_iter = 500
max_iter = 500
long_train = False
if long_train:
init_max_iter *= 4
max_iter *= 4
# compute validation accuracy every val_step epochs
init_step = 100
val_step = 25
# early stopping for val models, init via weight propagation
es_val = True
# early stopping for final models, initialized with val weights
es_fin = True
# tol == -1 -> train until max_iter
tol_init = -1
tol_val = 1e-200 if es_val else -1
tol_fin = 1e-200 if es_fin else -1
# use a small percentage of training data for early stopping
es_percent = 0.1
# set early stopping=True and disable on case-by-case basis
mlpparms = dict(hidden_layer_sizes=layers, activation='relu', solver='adam',
alpha=1e-5, batch_size=batch_size, learning_rate='adaptive',
learning_rate_init=0.001, power_t=0.5, max_iter=max_iter,
random_state=42, early_stopping=True, tol=tol_init,
warm_start=False, momentum=0.9, nesterovs_momentum=True,
shuffle=False, validation_fraction=es_percent,
beta_1=0.9, beta_2=0.999, epsilon=1e-10, verbose=False,
n_iter_no_change=10)
model = MLP(**mlpparms)
# map state parameter values to unit range
inputs = normalize_inputs(inputs,names,clip=False)
val_loss = dict(mse=[],mae=[])
# pick a few validation toa spectra at varying mad to show in detail
val_tgts = tgts[val_idx]
val_dtgts = np.diff(val_tgts,1)
val_dtgts_med = np.median(val_dtgts,axis=0)
val_dtgts_mad = abs(val_dtgts-val_dtgts_med).sum(axis=1)
sorti = np.argsort(val_dtgts_mad)
val_qper = [0.25,0.5,0.75]
val_toaidx = [sorti[int(n_samplesv*qp)] for qp in val_qper]
val_toalab = ['argq%d'%(qi*100) for qi in val_qper]
val_toalab = dict(zip([sorti[0]]+val_toaidx+[sorti[-1]],
['argmin']+val_toalab+['argmax']))
for ridx,rlab in val_toalab.items():
print('val_dtgts_mad[%s]: "%s"'%(rlab,str((val_dtgts_mad[ridx]))))
print('inputs[%s]: "%s"'%(rlab,str((inputs[val_idx[ridx]]))))
# bookkeeping for selected val_toa spectra + predictions
val_toatrue = dict([(idx,s.zeros(n_wl)) for idx in val_toaidx])
val_toapred = dict([(idx,s.zeros(n_wl)) for idx in val_toaidx])
val_toamse = dict([(idx,s.zeros(n_wl)) for idx in val_toaidx])
plot_toadat=False
if plot_toadat:
dwl=wl[:val_dtgts.shape[1]]
fig0,ax0 = pl.subplots(2,1,sharex=True,sharey=False)
for ridx,rlab in val_toalab.items():
ax0[0].plot(wl,val_tgts[ridx],label=rlab)
ax0[1].plot(dwl,val_dtgts[ridx],label='diff(%s)'%rlab)
ax0[0].legend()
ax0[1].legend()
pl.show()
accum = s.zeros(n_wl)
modelclass = '_'.join(['mlp']+list(map(str,layers))+[str(n_over)])
if long_train:
modelclass += '_longtr'
if not use_wp:
modelclass += '_nowp'
modeldir = pathjoin(outdir,modelclass)
print('modeldir: "%s"'%str((modeldir)))
if not pathexists(modeldir):
os.makedirs(modeldir)
log_filename = 'train_pid%s.log'%str(os.getpid())
log_file = pathjoin(modeldir,log_filename)
print('Writing log_file=%s'%log_file)
sleep(1)
log_fid = open(log_file,'w')
print('# c, iter, mse, mae, time',file=log_fid)
modelprefix = pathjoin(modeldir,splitext(trainf)[0])
modelbase = modelprefix+'_c%s.pkl'
init_modelbase = modelprefix+'_init_c%s.pkl'
fin_modelbase = modelprefix+'_fin_c%s.pkl'
# weight/bias data bookeeping
W,b = {},{}
# weight/bias output files
Wf,bf = {},{}
# validation weights/biases constructed on training set,
# assesed on validation set
W['val'] = [np.zeros([n_wl,n_inputs,n_hidden]),
np.zeros([n_wl,n_hidden,n_over])]
b['val'] = [np.zeros([n_wl,n_hidden]),
np.zeros([n_wl,n_over])]
# "final" weights/biases constructed using *all* states
# NOTE: use these during deployment to isofit/rt_nn.py
W['fin'] = [np.zeros([n_wl,n_inputs,n_hidden]),
np.zeros([n_wl,n_hidden,n_over])]
b['fin'] = [np.zeros([n_wl,n_hidden]),
np.zeros([n_wl,n_over])]
# output file paths for validation/final outputs
# TODO (BDB, 04/17/19): make multi_layer consistent
Wf['val'] = [modelprefix+'_W%dv.npy'%l for l in [1,2]]
bf['val'] = [modelprefix+'_b%dv.npy'%l for l in [1,2]]
Wf['fin'] = [modelprefix+'_W%dnpy'%l for l in [1,2]]
bf['fin'] = [modelprefix+'_b%d.npy'%l for l in [1,2]]
if n_layers==3:
Wf['val'] = [Wf['val'][0],modelprefix+'_Wmv.npy',Wf['val'][1]]
bf['val'] = [bf['val'][0],modelprefix+'_bmv.npy',bf['val'][1]]
Wf['fin'] = [Wf['fin'][0],modelprefix+'_Wm.npy',Wf['fin'][1]]
bf['fin'] = [bf['fin'][0],modelprefix+'_bm.npy',bf['fin'][1]]
W['val'] = [W1v[0],np.zeros([n_wl,n_hidden,n_hidden]),W2v[1]]
b['val'] = [b1v[0],np.zeros([n_wl,n_hidden]),b2v[1]]
W['fin'] = [W1f[0],np.zeros([n_wl,n_hidden,n_hidden]),W2f[1]]
b['fin'] = [b1f[0],np.zeros([n_wl,n_hidden]),b2f[1]]
abserr = dict(fin=np.zeros([n_samples,len(wl)]), val=np.zeros([n_samplesv,len(wl)]))
sqderr = dict(fin=np.zeros([n_samples,len(wl)]), val=np.zeros([n_samplesv,len(wl)]))
# initialize model on initial channel(s)
cr = s.arange(0,n_over)
tr_input = inputs[tr_idx]
val_input = inputs[val_idx]
tr_tgts = tgts[tr_idx,cr]
val_tgts = tgts[val_idx,cr]
init_model_file = init_modelbase%str(0)
model_params = model.get_params(deep=True)
model_init = None
model_init_time = gettime()
if not pathexists(init_model_file) or retrain:
fit_init = fit(model, init_model_file, tr_input, tr_tgts, val_input,
val_tgts, init_max_iter, es_tol=tol_init,
val_step=init_step)
model_init,model_init_err = fit_init
model_init_mse,model_init_mae = model_init_err
model_init_time = gettime()-model_init_time
model_init_iter = model_init.n_iter_
print('%d, %d, %.16f, %.16f, %d'%(-1,model_init_iter,model_init_mse,
model_init_mae,model_init_time),
file=log_fid)
log_fid.flush()
else:
model_init = joblib.load(init_model_file)
print('loaded',init_model_file)
models = [model_init]
model_file = modelbase%str(0)
# train channelwise subnetworks
for c in range(n_wl):
# define channel range (cmin==cmax for monochromatic)
n_off = n_wl-c
if c < n_half+1:
cmin,cmax = 0,n_over-1
elif n_off < n_half+1:
cmin,cmax = n_wl-n_over,n_wl-1
else:
cmin,cmax = c-n_half,c+n_half
cr = s.arange(cmin,cmax+1) if (n_half==0 or average_over) else c
print('\n##### Training subnetwork for center channel %d, wl=[%.2f,%.2f] ##########'%(c,wl[cmin],wl[cmax]))
# grab the training/validation targets for the current channel(s)
fin_tgts = tgts[:,cr].reshape([-1,n_over])
tr_tgts = tgts[tr_idx,cr].reshape([-1,n_over])
val_tgts = tgts[val_idx,cr].reshape([-1,n_over])
# update model file after storing the previous file
prev_model_file = None
if use_wp:
prev_model_file = model_file if c!=0 else init_model_file
model_file = modelbase%str(c)
model_c_time = gettime()
if not pathexists(model_file) or retrain:
# train new model, save to model_file
model_c = clone(model)
model_c.set_params(**model_params)
fit_c = fit(model, model_file, tr_input, tr_tgts, val_input,
val_tgts, max_iter, val_step=val_step,
es_tol=tol_val, prev_model_file=prev_model_file)
model_c,model_c_err = fit_c
model_c_mse,model_c_mae = model_c_err
model_c_iter = model_c.n_iter_
model_c_time = gettime()-model_c_time
else:
# restore from model_file
model_c = joblib.load(model_file)
val_preds = model_c.predict(val_input).reshape(val_tgts.shape)
model_c_iter = model_c.n_iter_
model_c_mse = mse(val_tgts,val_preds)
model_c_mae = mae(val_tgts,val_preds)
model_c_time = gettime()-model_c_time
print('%d, %d, %.16f, %.16f, %d'%(c,model_c_iter,model_c_mse,model_c_mae,
model_c_time),file=log_fid)
log_fid.flush()
if c==0 and model_init is not None:
# replace initial model with refined version
models[0] = model_c
else:
models.append(model_c)
# save the output validation weights / biases
for l in range(n_layers):
W['val'][l][c] = np.array(model_c.coefs_[l])
b['val'][l][c] = np.array(model_c.intercepts_[l])
val_preds = model_c.predict(val_input).reshape(val_tgts.shape)
#print('cr,val_preds.shape,val_tgts.shape: "%s"'%str((cr,val_preds.shape,
# val_tgts.shape)))
# generate detailed summaries for val_toaidx spectra
for ri in val_toaidx:
if n_half==0 or average_over:
# model generates predictions for 0 or more adjacent channels and
# we average the predictions for all generated channels
predi,truei = val_preds[ri],val_tgts[ri]
else:
# model generates predictions for 1 or more adjacent channels but
# we only consider the target channel in generating output
# target channel centered unless c first/prev channel
predi,truei = val_preds[ri][c-cmin],val_tgts[ri][c-cmin]
val_toaabserri = np.abs(predi-truei)
val_toatrue[ri][cr] = truei
val_toapred[ri][cr] += predi
val_toamse[ri][cr] += (val_toaabserri*val_toaabserri)
# keep track of how many times we generate a prediction for each channel
# (accmum==ones(n_wvl) for monochromatic case)
accum[cr] += 1
val_abserr = np.abs(val_tgts-val_preds)
val_sqderr = val_abserr*val_abserr
sqderr['val'][:,cr] += val_sqderr
abserr['val'][:,cr] += val_abserr
# track the mean/std/median/mad of the mse and mae
val_loss['mse'].append([np.mean(val_sqderr),np.std(val_sqderr),
np.median(val_sqderr),mad(val_sqderr)])
val_loss['mae'].append([np.mean(val_abserr),np.std(val_abserr),
np.median(val_abserr),mad(val_abserr)])
if val_only: # skip training "final" model
print('val_only==True, skipping production model fit')
continue
print('\n##### Training full subnetwork for center channel %d, wl=[%.3f,%.3f] ##########'%(c,wl[cmin],wl[cmax]))
fin_model_file = fin_modelbase%(str(c))
fin_model_filename = pathsplit(fin_model_file)[1]
model_f_time = gettime()
if not pathexists(fin_model_file) or retrain:
# train "final" production model on *all* inputs
# start with converged validation model for this channel
model_f = deepcopy(model_c)
model_f.fit(inputs, fin_tgts)
model_f_time = gettime()-model_f_time
joblib.dump(model_f, fin_model_file)
print('saved',fin_model_filename)
else:
model_f = joblib.load(fin_model_file)
print('loaded',fin_model_filename)
fin_preds = model_f.predict(inputs).reshape(fin_tgts.shape)
fin_abserr = np.abs(fin_tgts-fin_preds)
fin_sqderr = fin_abserr*fin_abserr
sqderr['fin'][:,cr] += fin_sqderr
abserr['fin'][:,cr] += fin_abserr
# extract the final model weights / biases
for l in range(n_layers):
W['fin'][l][c] = np.array(model_f.coefs_[l])
b['fin'][l][c] = np.array(model_f.intercepts_[l])
# end of channelwise training loop
evalkeys = ['val'] if val_only else ['val','fin']
for evalkey in evalkeys:
abserr[evalkey] = abserr[evalkey] / accum
sqderr[evalkey] = sqderr[evalkey] / accum
outmat = modelprefix+'_%s_sqderr.mat'%evalkey
savemat(outmat,{'training_idx':tr_idx,
'validation_idx':val_idx,
'sqderr':sqderr[evalkey],
'abserr':abserr[evalkey],
'wl':wl})
print('saved',outmat)
Woutf,boutf = Wf[evalkey],bf[evalkey]
Wout,bout = W[evalkey],b[evalkey]
for l,(Wl,bl) in enumerate(zip(Woutf,boutf)):
np.save(Wl,Wout[l])
np.save(bl,bout[l])
print('saved %s weights to: "%s"'%(evalkey,str(((Woutf[0],boutf[0]),(Woutf[1],boutf[1])))))
# plot the selected val_toaidx predictions vs. actuals
for idx in val_toaidx:
toatitle = ', '.join('%s=%.2f'%(n,v) for n,v in zip(names,inputs[idx]))
toatrue = val_toatrue[idx]
toapred = val_toapred[idx] / accum
toamse = val_toamse[idx] / accum
toafig = modelprefix+'_toa%d_%s_pred.pdf'%(val_idx[idx],
val_toalab[idx])
fig,ax = pl.subplots(3,1,sharex=True,sharey=False)
ax[0].plot(wl,toatrue)
ax[0].plot(wl,toapred,c='r',ls=':')
ax[1].plot(wl[:-1],s.diff(toatrue))
ax[1].plot(wl[:-1],s.diff(toapred),c='r',ls=':')
ax[2].plot(wl,toamse)
pl.suptitle(toatitle)
pl.savefig(toafig)
print('saved toafig: "%s"'%str((toafig)))
# plot mse/mae error curves
for errkey in val_loss:
errdat = np.array(val_loss[errkey])
errmean,errstd = errdat[:,0],errdat[:,1]
errstr='mean val_%s: %g, std: %g'%(errkey,s.mean(errdat[:,0]),s.std(errdat[:,0]))
print(errstr)
errfig = modelprefix+'_%s.pdf'%errkey
fig,ax = pl.subplots(2,1,sharex=True,sharey=False)
plotmeanstd(ax[0],wl,errmean,errstd)
plotmeanstd(ax[1],wl,errmean,errstd,diff=True)
ax[0].set_ylabel(errkey)
ax[1].set_ylabel('diff(%s)'%errkey)
ax[0].set_title(errstr)
pl.savefig(errfig)
print('saved errfig: "%s"'%str((errfig)))
# free some memory
del D
MSE = np.sum((sample_validation.rings - prediccions)**2) / N_valid
if mse is None or mse > MSE:
size_ideal_mse = size
mse = MSE
mses.append(MSE)
size = size_ideal_mse
print("Best config with {}".format(size))
model_nnet = MLPRegressor(hidden_layer_sizes=size,
alpha=0,
activation="logistic",
max_iter=1000,
solver='lbfgs')
model_nnet = model_nnet.fit(sample.loc[:, "male":"shell_weight"], sample.rings)
print(model_nnet.get_params(), file=open('coeficients/mlp_singlelayer', 'w'))
prediccions = model_nnet.predict(sample.loc[:, "male":"shell_weight"])
#Obtenim R-Squared sobre train
NMSE = sum((sample.rings - prediccions)**2) / ((N - 1) * np.var(sample.rings))
print("NMSE MLP:", NMSE)
R_squared = (1 - NMSE) * 100
print("Our model explain the {}% of the train variance".format(R_squared))
#Obtenim les mètriques que ens serviran per comparar els diferents models
prediccions = model_nnet.predict(sample_validation.loc[:,
"male":"shell_weight"])
MAE = np.sum(abs(sample_validation.rings - prediccions)) / N_valid
print("MAE on validation data:", MAE)
MSE_valid = np.sum((sample_validation.rings - prediccions)**2) / N_valid
print("MSE validation MLP:", MSE_valid)
price_vec.append(l[3])
scaler = preprocessing.MinMaxScaler(feature_range=(-1, 1))
mat = numpy.array(feature_mat)
mat = scaler.fit_transform(mat)
price_vec = numpy.array(price_vec)
#ftrain, ftest, ptrain, ptest = train_test_split(mat, price_vec, test_size=0.1)
clf = MLPRegressor(solver='lbfgs',
hidden_layer_sizes=(4, 4, 4),
random_state=1)
clf.fit(ftrain, ptrain)
result = clf.predict(ftest)
print(result)
print(len(result))
print(ptest)
print(len(ptest))
print('score sklearn :', clf.score(ftest, ptest))
print(clf.get_params())
# pos = where(sign_vec == 1)
# neg = where(sign_vec == 0)
# scatter(mat[pos, 0], mat[pos, 1], marker='o', c='b')
# scatter(mat[neg, 0], mat[neg, 1], marker='x', c='r')
# xlabel('Exam 1 score')
# ylabel('Exam 2 score')
# legend(['Not Admitted', 'Admitted'])
# show()
def _mlp_regression_train(table,
feature_cols,
label_col,
hidden_layer_sizes=(100, ),
activation='relu',
solver='adam',
alpha=0.0001,
batch_size_auto=True,
batch_size='auto',
learning_rate='constant',
learning_rate_init=0.001,
max_iter=200,
random_state=None,
tol=0.0001):
_, features = check_col_type(table, feature_cols)
label = table[label_col]
mlp_model = MLPRegressor(hidden_layer_sizes=hidden_layer_sizes,
activation=activation,
solver=solver,
alpha=alpha,
batch_size=batch_size,
learning_rate=learning_rate,
learning_rate_init=learning_rate_init,
max_iter=max_iter,
shuffle=True,
random_state=random_state,
tol=tol)
mlp_model.fit(features, label)
predict = mlp_model.predict(features)
intercepts = mlp_model.intercepts_
coefficients = mlp_model.coefs_
loss = mlp_model.loss_
_mean_absolute_error = mean_absolute_error(label, predict)
_mean_squared_error = mean_squared_error(label, predict)
_r2_score = r2_score(label, predict)
result_table = pd.DataFrame.from_items(
[['Metric', ['Mean Absolute Error', 'Mean Squared Error', 'R2 Score']],
['Score', [_mean_absolute_error, _mean_squared_error, _r2_score]]])
label_name = {
'hidden_layer_sizes': 'Hidden Layer Sizes',
'activation': 'Activation Function',
'solver': 'Solver',
'alpha': 'Alpha',
'batch_size': 'Batch Size',
'learning_rate': 'Learning Rate',
'learning_rate_init': 'Learning Rate Initial',
'max_iter': 'Max Iteration',
'random_state': 'Seed',
'tol': 'Tolerance'
}
get_param = mlp_model.get_params()
param_table = pd.DataFrame.from_items(
[['Parameter', list(label_name.values())],
['Value', [get_param[x] for x in list(label_name.keys())]]])
rb = BrtcReprBuilder()
rb.addMD(
strip_margin("""
| ### MLP Classification Result
| {result}
| ### Parameters
| {list_parameters}
""".format(result=pandasDF2MD(result_table),
list_parameters=pandasDF2MD(param_table))))
model = _model_dict('mlp_regression_model')
model['features'] = feature_cols
model['label'] = label_col
model['intercepts'] = mlp_model.intercepts_
model['coefficients'] = mlp_model.coefs_
model['loss'] = mlp_model.loss_
model['mean_absolute_error'] = _mean_absolute_error
model['mean_squared_error'] = _mean_squared_error
model['r2_score'] = _r2_score
model['activation'] = activation
model['solver'] = solver
model['alpha'] = alpha
model['batch_size'] = batch_size
model['learning_rate'] = learning_rate
model['learning_rate_init'] = learning_rate_init
model['max_iter'] = max_iter
model['random_state'] = random_state
model['tol'] = tol
model['mlp_model'] = mlp_model
model['_repr_brtc_'] = rb.get()
return {'model': model}
#poly = make_pipeline(PolynomialFeatures(3), Ridge())
mpl = MLPRegressor(beta_1=0.99)
'''
y_t = y[-1000:-2]
y = y[0:-1000]
X_t = X[-1000:-2]
X = X[0:-1000]
mpl.fit(X, y)
poly.fit(X, y)
mpl_pred = mpl.predict(X_t)
poly_pred = poly.predict(X_t)
'''
mpl_pred = cross_val_predict(mpl, X, y, cv=10)
#poly_pred = cross_val_predict(poly, X, y, cv=10)
#nn_pred = cross_val_predict(model, X, y, cv=10)
print mpl.get_params()
def plot_cross():
fig, ax = plt.subplots()
ax.scatter(y, mpl_pred, c='b', marker='x')
# ax.scatter(y, poly_pred, c='y', marker ='+')
#ax.scatter(y, nn_pred, c='r')
ax.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=4)
ax.set_xlabel('Measured')
ax.set_ylabel('Predicted')
plt.show()
#mpl_pred = shift(mpl_pred, 30, cval=0)
#poly_pred = shift(poly_pred, 30, cval=0)
def plot_time():
from sklearn import ensemble
clf4=ensemble.RandomForestRegressor(random_state=0)
clf4.fit(input.T[0:384],targets[0:384])
print(clf4.score(input.T[384:],targets[384:]))
import matplotlib.pyplot as plt
parameters={ 'min_samples_split':[2,10], 'max_depth':[3, 15],'n_estimators':[10,50]}
from sklearn import grid_search
###### Step 5 Testing With Neural Networks
from sklearn.neural_network import MLPRegressor
clf5=MLPRegressor(random_state=0,hidden_layer_sizes=500,activation='logistic',max_iter=500,)
clf5.fit(input.T[0:384],targets[0:384])
pred=clf5.predict(input.T[384:])
pred_train=clf5.predict(input.T[0:384])
print(clf5.score(input.T[384:],targets[384:]))
print(clf5.get_params())
### Plotting the results
plt.figure(1)
plt.plot(range(0,len(targets[384:])),pred,'red',range(0,len(targets[384:])),targets[384:],'blue')
plt.figure(2)
plt.plot(range(0,len(targets[0:384])),pred_train,'red',range(0,len(targets[0:384])),targets[0:384],'blue')
plt.show()
from sklearn.externals import joblib
#### Dumping the obtained classifier
joblib.dump(clf5, 'clf5.pkl')
predict_train1 = mlp.predict(data_train)
rmse_train1 = rmse(target_train, predict_train1)
#RMSE Test Data
predict_test1 = mlp.predict(data_test)
rmse_test1 = rmse(target_test, predict_test1)
#Weight values
weights = mlp.coefs_
#Bias values
biases = mlp.intercepts_
#Write Params down
results = open("diabetesBaseResults.txt", "a")
results.write("----PARAMS----\n")
results.write(str(mlp.get_params()) + "\n" + "\n")
#Write RMSE down
results = open("diabetesBaseResults.txt", "a")
results.write("----RMSE----\n")
results.write(" Train Data | Test Data " + "\n")
results.write("Pre-Train " + str(rmse_train0) + " " + str(rmse_test0) +
"\n")
results.write("Post-Train " + str(rmse_train1) + " " + str(rmse_test1) +
"\n" + "\n")
#Write down target values and actual values for TEST data
results = open("diabetesBaseResults.txt", "a")
results.write("----TARGETS----PREDICTED VALS----\n")
for elem in range(len(target_test)):
results.write(
for lri in parameters["learning_rate_init"]:
for lr in parameters["learning_rate"]:
for alpha in parameters["alpha"]:
for hls in parameters["hidden_layer_sizes"]:
clf = MLPRegressor(solver='lbfgs',
learning_rate_init=lri,
learning_rate=lr,
alpha=alpha,
hidden_layer_sizes=hls,
early_stopping=True,
random_state=42)
clf.fit(X_train, T_train)
T_pred = clf.predict(X_val)
if mean_squared_error(T_val, T_pred) < min_mse:
min_mse = mean_squared_error(T_val, T_pred)
best_estimator = clf.get_params()
print(
f'The best paramters returned by the grid search are:\n {best_estimator}'
)
print(f'The minimal MSE on the validation set is: {min_mse}')
# Fit the best estimator
clf = MLPRegressor(**best_estimator)
clf.fit(X_train, T_train)
# Predict on the test set
T_pred = clf.predict(X_test)
# Plot T_pred against T_test
plot_mg_time_series([T_test, T_pred],
solver=solver,
random_state=random_state)
# training the model
mlpr.fit(x_train, y_train)
# looking at the attributes
loss = mlpr.loss_ # the loss computed with the loss function
coefs = mlpr.coefs_ # a list of the weight matrix for each layer
intercepts = mlpr.intercepts_ # a list of the bias vector for each layer
n_iter = mlpr.n_iter_ # the number of interations the solver has run
n_layers = mlpr.n_layers_ # the number of layers of the model
n_outputs = mlpr.n_outputs_ # the number of outputs
out_activation = mlpr.out_activation_ # the name of the output activation function used
# looking at the methods
get_params = mlpr.get_params() # returning the parameters for the model
predition_array = mlpr.predict(
x_test
) # running the test dataset through the model, giving an array of predicted values
train_score = mlpr.score(
x_train, y_train) # returns the mean accuracy of the training set
test_score = mlpr.score(x_test,
y_test) # returns the mean accuracy of the test set
print(
'The R^2 score for the train dataset is: %.3f and the R^2 for the test dataset is: %.3f'
% (train_score, test_score))
pdb.set_trace()
train = values[:n_train_hours, :]
test = values[n_train_hours:-30, :]
# split into input and outputs
n_obs = n_hours * n_features
train_X, train_y = train[:, :-5], train[:, -1]
test_X, test_y = test[:, :-5], test[:, -1]
print(train_X.shape, len(train_X), train_y.shape)
eval_set = [(test_X, test_y)]
# fit model no training data
####learning_rate={0.01,0.05,0.1,0.2}
####max_math={3,4,5}
####n_estimators={80,90,100,110,120}
model = MLPRegressor()
model.fit(train_X, train_y)
print(model.get_params())
####对模型进行打分
preds_train = model.predict(train_X)
print("模型打分情况：", model.score(train_X, train_y))
yhat = model.predict(test_X)
# invert scaling for forecast
yhat = yhat.reshape(len(yhat), 1)
inv_yhat = concatenate((test_X[:, -31:], yhat), axis=1)
inv_yhat = scaler.inverse_transform(inv_yhat)
inv_yhat = inv_yhat[:, -1]
# invert scaling for actual
test_y = test_y.reshape((len(test_y), 1))
inv_y = concatenate((test_X[:, -31:], test_y), axis=1)
inv_y = scaler.inverse_transform(inv_y)
print("Coeficiente de R^2 asociado a la predicción dentro de la muestra")
print(regr.score(X_train,y_train))
en_pred = regr.predict(X_test)
print("R^2 para el conjunto de test", regr.score(X_test,y_test))
print("Error absoluto medio", mean_absolute_error(y_test,en_pred))
input("\n--- Pulsa una tecla para continuar ---\n")
print("Redes neuronales")
# Establezco un número alto de iteraciones máximas para que no haya problemas de convergencia
redes = MLPRegressor(max_iter = 2000)
redes.fit(X_train,y_train)
print("Parámetros de la red neuronal")
print(redes.get_params())
# R^2 para el conjunto de training
print("In R^2")
print(redes.score(X_train,y_train))
red_pred = redes.predict(X_test)
# R^2 para el conjunto de test
print("Out R^2")
print(redes.score(X_test,y_test))
print("Error absoluto medio", mean_absolute_error(y_test,red_pred))
input("\n--- Pulsa una tecla para continuar ---\n")
print("SVR")
svr = SVR(C=1,epsilon=0.2)
#clf = MLPRegressor(solver='lbfgs', alpha=1e-5,
# hidden_layer_sizes=(50), random_state=1, max_iter=1000)
#pipeline = PMMLPipeline([
# ('clf', clf)
# ])
#pipeline.fit(df[df.columns.difference(["Open"])], df["Prediction"])
#sklearn2pmml(pipeline, "PredictiveModel.pmml")
estimator = MLPRegressor(hidden_layer_sizes=(1000, ) * 4,
activation='relu',
max_iter=int(1e2),
verbose=True,
random_state=1,
tol=0)
logging.info('Estimator: {}'.format(estimator.get_params()))
y_pred = sklearn.model_selection.cross_val_predict(estimator=estimator,
X=X,
y=y,
cv=6)
estimator.fit(X_train, y_train)
confidence = estimator.score(X_test, y_test)
print("confidence: ", confidence)
forecast_prediction = estimator.predict(X_forecast)
print(forecast_prediction)
z = y_pred.tolist()
price1 = pd.concat([pd.Series(df['Open']), pd.Series(z)],
def cascade_prediction(train_file, test_file, val_file, embedding_file,
embedding_dim):
embeddings = np.fromfile(embedding_file,
np.float32).reshape(-1, embedding_dim)
#embeddings = read_embeddings_avg(embedding_file)
label = 0
x_test = []
y_test = []
index = 0
rb = open(test_file, 'r')
for line in rb.readlines():
elems = line.strip().split('\t')
x_test.append(embeddings[index])
index += 1
value = int(elems[-1].split(' ')[label])
value = np.log(value + 1.0) / np.log(2.0)
y_test.append(value)
rb.close()
x_train = []
y_train = []
rb = open(train_file, 'r')
for line in rb.readlines():
elems = line.strip().split('\t')
x_train.append(embeddings[index])
index += 1
value = int(elems[-1].split(' ')[label])
value = np.log(value + 1) / np.log(2.0)
y_train.append(value)
rb.close()
x_val = []
y_val = []
rb = open(val_file, 'r')
for line in rb.readlines():
elems = line.strip().split('\t')
x_val.append(embeddings[index])
index += 1
value = int(elems[-1].split(' ')[label])
value = np.log(value + 1) / np.log(2.0)
y_val.append(value)
rb.close()
parameter_space = {
'hidden_layer_sizes': [(64, 32, 16), (64, 32, 64), (64, )],
'activation': ['tanh', 'relu', 'logistic'],
'solver': ['sgd', 'adam'],
'alpha': [0.0001, 0.001, 0.05, 0.01, 0.5, 1],
'learning_rate': ['constant', 'adaptive'],
'shuffle': [False],
'random_state': [0],
'max_iter': [200]
}
x = x_train + x_val
y = y_train + y_val
fold = [-1 for i in x_train]
fold.extend([0 for i in x_val])
'''
model = MLPRegressor()
ps = PredefinedSplit(test_fold=fold)
clf = GridSearchCV(model, parameter_space, n_jobs=10, cv=ps)
clf.fit(x, y)
'''
clf = MLPRegressor(hidden_layer_sizes=(64, 32),
activation=relu,
solver=adam,
shuffle=False,
random_state=0)
clf.fit(x, y)
print(clf.get_params())
y_pred = clf.predict(x_test)
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))
