def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
print len(X[0])
mean, std = normalize(X)
y = assemble_y(training)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=325,
dropout1_p=0.4,
hidden2_num_units=325,
dropout2_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=1),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=85,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
def train(trainfile, weatherfile):
weather = load_weather(weatherfile)
training = load_training(trainfile)
np.random.seed(42)
X = assemble_X(training, weather)
mean, std = normalize(X)
y = assemble_y(training)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=400,
dropout1_p=0.4,
hidden2_num_units=200,
dropout2_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=4),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
max_epochs=60,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:, 0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
# Compute the z-scores for both train and validation. However, use mean and standard deviation for training
# on both. This is customary because we trained on this standard deviation and mean. Additionally, our
# prediction set might too small to calculate a meaningful mean and standard deviation.
X_train_z = zscore(X_train, train_mean, train_sdev) #scipy.stats.mstats.zscore(X_train)
X_validate_z = zscore(X_validate, train_mean, train_sdev) #scipy.stats.mstats.zscore(X_validate)
#These can be used to check my zscore calc to numpy
#print(X_train_z)
#print(scipy.stats.mstats.zscore(X_train))
# Provide our own validation set
def my_split(self, X, y, eval_size):
return X_train_z,X_validate_z,y_train,y_validate
net0.train_test_split = types.MethodType(my_split, net0)
# Train the network
net0.fit(X_train_z,y_train)
# Predict the validation set
pred_y = net0.predict(X_validate_z)
# Display predictions and count the number of incorrect predictions.
species_names = ['setosa','versicolour','virginica']
count = 0
wrong = 0
for element in zip(X_validate,y_validate,pred_y):
print("Input: sepal length: {}, sepal width: {}, petal length: {}, petal width: {}; Expected: {}; Actual: {}".format(
element[0][0],element[0][1],element[0][2],element[0][3],
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
mean, std = normalize(X)
y = assemble_y(training)
col_labels = ["date.year","date.month","date.day","block","lat","long",]
for obs in ["Tmax","Tmin","Tavg","DewPoint","WetBulb","PrecipTotal","Depart","Sunrise","Sunset","Heat","Cool","ResultSpeed","ResultDir"]:
for day in ["1","2","3","5","8","12"]:
col_labels.append(obs+"_"+day)
for i_spec in range(6):
col_labels.append("species_"+str(i_spec))
X_file = csv.writer(open("X.csv", "w"))
X_file.writerow(col_labels)
for row in X:
X_file.writerow(row)
y_file = csv.writer(open("y.csv", "w"))
y_file.writerow(["y"])
for row in y:
y_file.writerow(row)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=600,
dropout1_p=0.5,
hidden2_num_units=400,
dropout2_p=0.5,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=20),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=500,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
output_num_units=4,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.899,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=4),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=75,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(Xtrh, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
def NN1_Classifier(Train_DS, y, Actual_DS, Sample_DS, grid):
print("***************Starting NN1 Classifier***************")
t0 = time()
if grid:
#used for checking the best performance for the model using hyper parameters
print("Starting model fit with Grid Search")
else:
#y = y.reshape((-1, 1))
y = y.astype('int32')
Actual_DS = np.array(Actual_DS.astype('float32'))
Train_DS = np.array(Train_DS.astype('float32'))
learning_rate = theano.shared(np.float32(0.1))
#Define Model parms - 2 hidden layers
clf = NeuralNet(
layers=[
('input', InputLayer),
('dropout0', DropoutLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, Train_DS.shape[1]),
dropout0_p=0.25,
hidden1_num_units=400,
dropout1_p = 0.4,
hidden2_num_units=400,
dropout2_p = 0.4,
output_nonlinearity=softmax, # output layer uses identity function
output_num_units=5,
#optimization method
#update=sgd,
#update=nesterov_momentum,
update=adagrad,
update_learning_rate=0.01,
use_label_encoder=False,
batch_iterator_train=BatchIterator(batch_size=100),
#update_momentum=0.1,
# on_epoch_finished=[
# AdjustVariable('update_learning_rate', start=0.1, stop=0.0001),
# EarlyStopping(patience=10),
# ],
eval_size = 0.1,
regression=False,
max_epochs=60,
verbose=1
)
Train_DS, y = shuffle(Train_DS, y, random_state=42)
clf.fit(Train_DS, y)
_, X_valid, _, y_valid = clf.train_test_split(Train_DS, y, clf.eval_size)
y_pred=clf.predict(X_valid)
score=quadratic_weighted_kappa(y_valid, y_pred)
print("Best score: %0.3f" % score)
#Predict actual model
pred_Actual = clf.predict(Actual_DS)
print("Actual NN1 Model predicted")
#Get the predictions for actual data set
preds = pd.DataFrame(pred_Actual, index=Sample_DS.id.values, columns=Sample_DS.columns[1:])
preds.to_csv(file_path+'output/Submission_Roshan_NN_Model_1.csv', index_label='id')
print("***************Ending NN1 Classifier***************")
return pred_Actual
update_momentum=0.9,
regression=False,
on_epoch_finished=[
EarlyStopping(patience=5)
],
verbose=1,
max_epochs=100)
X_train, y_train, X_val, y_val, X_test, y_test = load_dataset(True)
def my_split(self, X, y, eval_size):
return X_train,X_val,y_train,y_val
net0.train_test_split = types.MethodType(my_split, net0)
net0.fit(X_train, y_train)
y_predict = net0.predict(X_val)
count = 0
wrong = 0
for element in zip(X_val,y_val,y_predict):
if element[1] != element[2]:
wrong = wrong + 1
count = count + 1
print("Incorrect {}/{} ({}%)".format(wrong,count,(wrong/count)*100))
# on_epoch_finished=[
# AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
# AdjustVariable('update_momentum', start=0.9, stop=0.999),
# ],
#on_epoch_finished=[
# AdjustVariable(learning_rate, target=0, half_life=4),
#],
eval_size=0.2,
verbose=1,
max_epochs=32,
batch_iterator_train=BatchIterator(batch_size=64),
batch_iterator_test=BatchIterator(batch_size=64))
net0.fit(X, y)
X_train, X_valid, y_train, y_valid = net0.train_test_split(
X, y, net0.eval_size)
probas = net0.predict_proba(X_valid)[:, 0]
#probas.loc[(probas[0]<0.3).values,[0]] = 0.1
print("ROC score", metrics.roc_auc_score(y_valid, (probas)))
#yp = DataFrame(net0.predict_proba(X_test),columns=[ u'Class_1', u'Class_2', u'Class_3', u'Class_4', u'Class_5', u'Class_6', u'Class_7', u'Class_8', u'Class_9'])
pred_prob = pd.DataFrame(net0.predict(X_test))
print pred_prob
sample = pd.read_csv('/Users/IkkiTanaka/Documents/KDDCup/sampleSubmission.csv',
header=None)
preds = pd.concat([sample[0], pd.DataFrame(pred_prob)], axis=1)
preds.to_csv('/Users/IkkiTanaka/Documents/KDDCup/pred/xgb/predNN1.csv',
header=None,
index=False)
for i in xrange(0, 50):
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
mean, std = normalize(X)
y = assemble_y(training)
col_labels = [
"date.year",
"date.month",
"date.day",
"block",
"lat",
"long",
]
for obs in [
"Tmax", "Tmin", "Tavg", "DewPoint", "WetBulb", "PrecipTotal",
"Depart"
]:
for day in ["1", "3", "7", "14"]:
col_labels.append(obs + "_" + day)
for i_spec in range(6):
col_labels.append("species_" + str(i_spec))
X_file = csv.writer(open("X.csv", "w"))
X_file.writerow(col_labels)
for row in X:
X_file.writerow(row)
y_file = csv.writer(open("y.csv", "w"))
y_file.writerow(["y"])
for row in y:
y_file.writerow(row)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=256,
dropout1_p=0.4,
hidden2_num_units=256,
dropout2_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=4),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
max_epochs=32,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:, 0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
# on_epoch_finished=[
# AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
# AdjustVariable('update_momentum', start=0.9, stop=0.999),
# ],
#on_epoch_finished=[
# AdjustVariable(learning_rate, target=0, half_life=4),
#],
eval_size=0.2,
verbose=1,
max_epochs=32,
batch_iterator_train=BatchIterator(batch_size=64),
batch_iterator_test=BatchIterator(batch_size=64 )
)
net0.fit(X, y)
X_train, X_valid, y_train, y_valid = net0.train_test_split(X, y, net0.eval_size)
probas = net0.predict_proba(X_valid)[:,0]
#probas.loc[(probas[0]<0.3).values,[0]] = 0.1
print("ROC score", metrics.roc_auc_score(y_valid, (probas)))
#yp = DataFrame(net0.predict_proba(X_test),columns=[ u'Class_1', u'Class_2', u'Class_3', u'Class_4', u'Class_5', u'Class_6', u'Class_7', u'Class_8', u'Class_9'])
pred_prob = pd.DataFrame(net0.predict( X_test ))
print pred_prob
sample = pd.read_csv('/Users/IkkiTanaka/Documents/KDDCup/sampleSubmission.csv',header=None)
preds = pd.concat([sample[0],pd.DataFrame(pred_prob)],axis=1)
preds.to_csv('/Users/IkkiTanaka/Documents/KDDCup/pred/xgb/predNN1.csv' ,header=None,index=False)
for i in xrange(0,50):
