def NNet(X, y, test):
#rows = int(X.shape[0])
cols = int(X.shape[1])
net = NeuralNet(
layers=[
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('dropout1', layers.DropoutLayer),
('hidden2', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('hidden3', layers.DenseLayer),
('dropout3', layers.DropoutLayer),
('hidden4', layers.DenseLayer),
#('dropout4',layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape=(None, cols),
hidden1_num_units=70,
dropout1_p=0.4,
hidden2_num_units=50,
dropout2_p=0.3,
hidden3_num_units=30,
dropout3_p=0.3,
hidden4_num_units=15,
#dropout4_p = 0.2,
output_num_units=len(np.unique(y)),
output_nonlinearity=lasagne.nonlinearities.softmax,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=100,
verbose=1,
)
# net.load_params_from('w3')
# details = net.get_all_params()
# oldw = net.get_all_params_values()
skf = cross_validation.StratifiedKFold(y, n_folds=7)
blend_train = np.zeros(X.shape[0])
prediction = []
blend_test_j = np.zeros((test.shape[0], len(skf)))
for i, (train_index, cv_index) in enumerate(skf):
print "Fold:", i
X_train = X[train_index, ]
y_train = y[train_index]
X_cv = X[cv_index, ]
#y_cv = y[cv_index]
net.fit(X_train, y_train)
blend_train[cv_index] = net.predict_proba(X_cv)[:, 1]
blend_test_j[:, i] = net.predict_proba(test)[:, 1]
prediction = blend_test_j.mean(1)
return prediction
def fit(xTrain, yTrain, dense0_num=800, dropout_p=0.5, dense1_num=500, update_learning_rate=0.01,
update_momentum=0.9, test_ratio=0.2, max_epochs=20):
#update_momentum=0.9, test_ratio=0.2, max_epochs=20, train_fname='train.csv'):
#xTrain, yTrain, encoder, scaler = load_train_data(train_fname)
#xTest, ids = load_test_data('test.csv', scaler)
num_features = len(xTrain[0,:])
num_classes = 9
print num_features
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('dense1', DenseLayer),
('output', DenseLayer)]
clf = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=dense0_num,
dropout_p=dropout_p,
dense1_num_units=dense1_num,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=update_learning_rate,
update_momentum=update_momentum,
eval_size=test_ratio,
verbose=1,
max_epochs=max_epochs)
clf.fit(xTrain, yTrain)
ll_train = metrics.log_loss(yTrain, clf.predict_proba(xTrain))
print ll_train
return clf
def nn_level2(train_x, train_y, test_x):
'''
neural net proba predict for level 2
'''
num_classes = len(np.unique(train_y))
num_features = train_x.shape[1]
layers0 = [('input', InputLayer), ('dropoutf', DropoutLayer),
('dense0', DenseLayer), ('dropout', DropoutLayer),
('dense1', DenseLayer), ('dropout2', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(
layers=layers0,
input_shape=(None, num_features),
dropoutf_p=0.15,
dense0_num_units=1000,
dropout_p=0.25,
dense1_num_units=500,
dropout2_p=0.25,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=theano.shared(np.float32(0.01)),
# on_epoch_finished=[AdjustVariable('update_learning_rate', start=0.02, stop=0.016)],
max_epochs=18,
eval_size=0.2,
verbose=1,
)
net0.fit(train_x, train_y)
pred = net0.predict_proba(test_x).astype(np.float32)
return pred
class DeepCls(object):
def __init__(self,n_cats=2,input_size=1040,k=1):
self.net1 = NeuralNet(
layers=[
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape=(None, input_size),
hidden_num_units=300,
output_nonlinearity=softmax,
output_num_units=n_cats,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=False,
max_epochs=2500,
verbose=1,
)
self.k=k
def __call__(self,hog_desc):
hog_desc=np.expand_dims(hog_desc,0)
prob=self.net1.predict_proba(hog_desc)
prob=prob.flatten()
return prob[self.k]
def OptNN(d1, h1, d2, h2, d3, start, stop, max_epochs):
params2 = params.copy()
on_epoch = [AdjustVariable('update_learning_rate',
start = start, stop = stop),
AdjustVariable('update_momentum', start = .9, stop = .999)]
params2['dropout1_p'] = d1
params2['dropout2_p'] = d2
params2['dropout3_p'] = d3
params2['dropout4_p'] = d4
params2['hidden1_num_units'] = h1
params2['hidden2_num_units'] = h2
params2['hidden3_num_units'] = h3
params2['max_epochs'] = max_epochs
params2['on_epoch_finished'] = on_epoch
kcv = StratifiedKFold(Y, 5, shuffle = True)
res = np.empty((len(Y), len(np.unique(Y)))); i = 1
CVScores = []
for train_idx, valid_idx in kcv:
logger.info("Running fold %d...", i); i += 1
net = NeuralNet(**params2)
net.set_params(eval_size = None)
net.fit(X[train_idx], Y[train_idx])
res[valid_idx, :] = net.predict_proba(X[valid_idx])
CVScores.append(log_loss(Y[valid_idx], res[valid_idx]))
return -np.mean(CVScores)
def train(trainfile, weatherfile):
weather = load_weather(weatherfile)
training = load_training(trainfile)
np.random.seed(42)
X = assemble_X(training, weather)
mean, std = normalize(X)
y = assemble_y(training)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=400,
dropout1_p=0.4,
hidden2_num_units=200,
dropout2_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=4),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
max_epochs=60,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:, 0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
print len(X[0])
mean, std = normalize(X)
y = assemble_y(training)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=325,
dropout1_p=0.4,
hidden2_num_units=325,
dropout2_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=1),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=85,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
class Classifier(BaseEstimator):
def __init__(self):
self.net = None
self.label_encoder = None
def fit(self, X, y):
layers0 = [('input', InputLayer), ('dropoutf', DropoutLayer),
('dense0', DenseLayer), ('dropout0', DropoutLayer),
('dense1', DenseLayer), ('dropout1', DropoutLayer),
('dense2', DenseLayer), ('dropout2', DropoutLayer),
('dense3', DenseLayer), ('dropout3', DropoutLayer),
('output', DenseLayer)]
X = X.astype(theano.config.floatX)
self.label_encoder = LabelEncoder()
y = self.label_encoder.fit_transform(y).astype(np.int32)
self.scaler = StandardScaler()
X = self.scaler.fit_transform(X)
num_classes = len(self.label_encoder.classes_)
num_features = X.shape[1]
self.net = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dropoutf_p=0.15,
dense0_num_units=1024,
dropout0_p=0.5,
dense0_nonlinearity=rectify,
dense1_num_units=1024,
dropout1_p=0.15,
dense1_nonlinearity=rectify,
dense2_num_units=1024,
dropout2_p=0.15,
dense2_nonlinearity=rectify,
dense3_num_units=1024,
dropout3_p=0.15,
dense3_nonlinearity=rectify,
output_num_units=num_classes,
update=adagrad,
update_learning_rate=0.01,
eval_size=0.2,
verbose=1,
max_epochs=150)
self.net.fit(X, y)
return self
def predict(self, X):
X = X.astype(theano.config.floatX)
X = self.scaler.fit_transform(X)
return self.label_encoder.inverse_transform(self.net.predict(X))
def predict_proba(self, X):
X = X.astype(theano.config.floatX)
X = self.scaler.fit_transform(X)
return self.net.predict_proba(X)
def OptNN2(d0, d1,d2, d3, h1, h2, h3, me, ls, le):
h1, h2, h3 = int(h1), int(h2), int(h3);
me = int(me)
params = dict(
layers = [
('input', layers.InputLayer),
('dropout1', layers.DropoutLayer),
('hidden1', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('hidden2', layers.DenseLayer),
('dropout3', layers.DropoutLayer),
('hidden3', layers.DenseLayer),
('dropout4', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape = (None, 93),
dropout1_p = d0,
hidden1_num_units = h1,
dropout2_p = d1,
hidden2_num_units = h2,
dropout3_p = d2,
hidden3_num_units = h3,
dropout4_p = d3,
output_nonlinearity = softmax,
output_num_units = 9,
update = nesterov_momentum,
update_learning_rate = theano.shared(float32(l_start)),
update_momentum = theano.shared(float32(m_start)),
regression = False,
on_epoch_finished = [
AdjustVariable('update_learning_rate', start = ls,
stop = le, is_log = True),
AdjustVariable('update_momentum', start = m_start,
stop = m_stop, is_log = False),
],
max_epochs = me,
verbose = 1,
)
CVScores = []
res = np.empty((len(Y), len(np.unique(Y))))
kcv = StratifiedKFold(Y, 5, shuffle = True); i = 1
for train_idx, valid_idx in kcv:
logger.info("Running fold %d...", i); i += 1
net = NeuralNet(**params)
net.set_params(eval_size = None)
net.fit(X[train_idx], Y[train_idx])
res[valid_idx, :] = net.predict_proba(X[valid_idx])
CVScores.append(log_loss(Y[valid_idx], res[valid_idx]))
return -np.mean(CVScores)
def train_test(train, labels, test, weight_decay):
net = NeuralNet(
layers=[
('input', InputLayer),
('dropout0', DropoutLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('dense2', DenseLayer),
('dropout2', DropoutLayer),
('dense3', DenseLayer),
('dropout3', DropoutLayer),
('output', DenseLayer),
],
update=nesterov_momentum,
loss=None,
objective=partial(WeightDecayObjective, weight_decay=weight_decay),
regression=False,
max_epochs=600,
eval_size=0.1,
#on_epoch_finished = None,
#on_training_finished = None,
verbose=bool(VERBOSITY),
input_shape=(None, train.shape[1]),
output_num_units=NCLASSES,
dense1_num_units=700,
dense2_num_units=1000,
dense3_num_units=700,
dense1_nonlinearity=LeakyRectify(leakiness=0.1),
dense2_nonlinearity=LeakyRectify(leakiness=0.1),
dense3_nonlinearity=LeakyRectify(leakiness=0.1),
output_nonlinearity=softmax,
dense1_W=HeUniform(),
dense2_W=HeUniform(),
dense3_W=HeUniform(),
dense1_b=Constant(0.),
dense2_b=Constant(0.),
dense3_b=Constant(0.),
output_b=Constant(0.),
dropout0_p=0.1,
dropout1_p=0.6,
dropout2_p=0.6,
dropout3_p=0.6,
update_learning_rate=shared(float32(0.02)), #
update_momentum=shared(float32(0.9)), #
batch_iterator_train=BatchIterator(batch_size=128),
batch_iterator_test=BatchIterator(batch_size=128),
)
net.fit(train, labels)
return net.predict_proba(test)
def train(hidden_units, subimission=False):
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
X, y, test_size=0.2, random_state=561672)
if subimission:
X_train = X
y_train = y
layers0 = [('input', InputLayer), ('dense0', DenseLayer),
('gausian0', GaussianNoiseLayer), ('dense1', DenseLayer),
('dropout0', DropoutLayer), ('dense2', DenseLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=hidden_units[0],
dense1_num_units=hidden_units[1],
dense2_num_units=hidden_units[2],
dropout0_p=0.5,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=theano.shared(float32(0.03)),
update_momentum=theano.shared(float32(0.9)),
on_epoch_finished=[
AdjustVariable('update_learning_rate',
start=0.03,
stop=0.0001),
AdjustVariable('update_momentum',
start=0.9,
stop=0.999),
],
eval_size=0.2,
verbose=1,
max_epochs=15)
net0.fit(X_train, y_train)
if subimission:
make_submission(net0,
X_test_submission,
ids,
encoder,
name="nn_%f_%f.csv" %
(hidden_units[0], hidden_units[1]))
else:
y_prob = net0.predict_proba(X_test)
print(" -- Finished training.")
score = logloss_mc(y_test, y_prob)
print("Score %f" % score)
return score, net0
def fit_two_dense(xTrain,
yTrain,
dropout_in=0.2,
dense0_num=800,
dropout_p=0.5,
dense1_num=500,
update_learning_rate=0.008,
update_momentum=0.9,
test_ratio=0.2,
max_epochs=40):
#xTrain, yTrain, encoder, scaler = load_train_data(train_fname)
#xTest, ids = load_test_data('test.csv', scaler)
#xTrain, yTrain, encoder, scaler = load_train_data(train_fname)
#xTest, ids = load_test_data('test.csv', scaler)
num_features = len(xTrain[0, :])
num_classes = 9
print num_features
layers0 = [('input', InputLayer), ('dropoutin', DropoutLayer),
('dense0', DenseLayer), ('dropout', DropoutLayer),
('dense1', DenseLayer), ('output', DenseLayer)]
clf = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dropoutin_p=dropout_in,
dense0_num_units=dense0_num,
dropout_p=dropout_p,
dense1_num_units=dense1_num,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=update_learning_rate,
update_momentum=update_momentum,
eval_size=test_ratio,
verbose=1,
max_epochs=max_epochs)
clf.fit(xTrain, yTrain)
ll_train = metrics.log_loss(yTrain, clf.predict_proba(xTrain))
print ll_train
return clf
def calc_prob_bag(i, best_max_epochs, NNargs, X_all, y_all, X_test):
np.random.seed(111*(i+1)) # diffs random seed to diffs bagging
print('\n - Bag: %i ' % (i+1))
if best_max_epochs == 0:
print(' First fit to get optimal num of epochs...')
NNargs["max_epochs"] = 1000
NNargs["eval_size"] = 0.05 # just a small test set to derive optimal numepochs
NNargs["on_epoch_finished"][-1] = EarlyStopping(patience=25) # more patience
clf_bag = NeuralNet(**NNargs)
clf_bag.fit(X_all, y_all)
global GLOBrealnumepochs
best_max_epochs = GLOBrealnumepochs
print(' we will refit now with max epochs = %i' % best_max_epochs)
NNargs["max_epochs"] = best_max_epochs
NNargs["eval_size"] = 0.0001
NNargs["on_epoch_finished"][-1] = EarlyStopping(patience=1000) # kind of a infinite patience to let max epochs rule
clf_bag = NeuralNet(**NNargs)
clf_bag.fit(X_all, y_all)
probs_bags = clf_bag.predict_proba(X_test)
return probs_bags
def main(deepL, alignments):
S, X = make_xy(alignments, deepL['features'])
h = DictVectorizer(sparse=False)
X = h.fit_transform(X)
X = X.astype(floatX)
# normalize feature matrix
min_max_scaler = preprocessing.MinMaxScaler()
X = min_max_scaler.fit_transform(X)
clf = NeuralNet(
layers=model(deepL['input_nodes'], deepL['output_nodes']),
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=False,
max_epochs=100,
verbose=2,
)
clf.load_params_from(opt.path + "/model/model.pkl")
proba = clf.predict_proba(X)
preds = []
probs = []
deepL['Y_rev'].update({10000: "unclassified"})
for ipx, p in enumerate(proba):
px = np.argmax(p)
if p[px] > min_prob[deepL['Y_rev'][px]]:
preds.append(px)
probs.append(p[px])
else:
preds.append(10000)
probs.append(0)
return [[S[ix], deepL['Y_rev'][i], probs[ix]]
for ix, i in enumerate(preds)]
def train_dnn(train, train_y, test):
num_features = train.shape[1]
num_classes = len(list(set(train_y)))
layers0 = [('input', InputLayer),
('dropout0', DropoutLayer),
('dense0', DenseLayer),
('dropout1', DropoutLayer),
('dense1', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dropout0_p = 0.1, #theano.shared(float32(0.1)),
dense0_num_units= 5000,
dropout1_p= 0.3, #theano.shared(float32(0.5)),
dense1_num_units = 10000,
dropout2_p = 0.5,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
#update_learning_rate=0.003,
#update_momentum=0.9,
update_learning_rate = theano.shared(float32(0.001)),
update_momentum=theano.shared(float32(0.9)),
objective_loss_function = categorical_crossentropy,
train_split = TrainSplit(0.2),
verbose=1,
max_epochs=150,
on_epoch_finished=[
EarlyStopping(patience = 20),
AdjustVariable('update_learning_rate', start=0.001, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
]
)
net0.fit(train, train_y)
print('Prediction Complete')
pred1 = net0.predict_proba(test)
return pred1
def train(features, features_sub,label):
layers0 = [('input', InputLayer),
('dropout0', DropoutLayer),
('dense0', DenseLayer),
('dropout1', DropoutLayer),
('dense1', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, features.shape[1]),
dropout0_p = 0.18, #theano.shared(float32(0.1)),
dense0_num_units= 2000,
dropout1_p= 0.6, #theano.shared(float32(0.4)),
dense1_num_units= 4000,
dropout2_p = 0.9, #theano.shared(float32(0.7)),
output_num_units= len(set(label)),
output_nonlinearity=softmax,
update=nesterov_momentum,
update_momentum = 0.95, #theano.shared(float32(0.9)),
update_learning_rate = 0.002, #theano.shared(float32(0.01)),
#update_momentum=theano.shared(float32(0.9)),
train_split = TrainSplit(0.2),
verbose=1,
max_epochs = 40
# on_epoch_finished=[
# AdjustVariable('update_learning_rate', start=0.01, stop=0.005),
# #AdjustVariable('update_momentum', start=0.9, stop=0.999),
# #AdjustVariable('dropout0_p', start = 0.1, stop = 0.2),
# # AdjustVariable('dropout1_p', start = 0.45, stop = 0.6),
# # AdjustVariable('dropout2_p', start = 0.8, stop = 0.9)
# ]
)
features = np.array(features.values, dtype = np.float32)
net0.fit(features, np.array(label, dtype = np.int32) )
print('Prediction Complete')
results = net0.predict_proba(np.array(features_sub.values, dtype = np.float32))
return
def calc_prob_bag(i, best_max_epochs, NNargs, X_all, y_all, X_test):
np.random.seed(111 * (i + 1)) # diffs random seed to diffs bagging
print('\n - Bag: %i ' % (i + 1))
if best_max_epochs == 0:
print(' First fit to get optimal num of epochs...')
NNargs["max_epochs"] = 1000
NNargs[
"eval_size"] = 0.05 # just a small test set to derive optimal numepochs
NNargs["on_epoch_finished"][-1] = EarlyStopping(
patience=25) # more patience
clf_bag = NeuralNet(**NNargs)
clf_bag.fit(X_all, y_all)
global GLOBrealnumepochs
best_max_epochs = GLOBrealnumepochs
print(' we will refit now with max epochs = %i' % best_max_epochs)
NNargs["max_epochs"] = best_max_epochs
NNargs["eval_size"] = 0.0001
NNargs["on_epoch_finished"][-1] = EarlyStopping(
patience=1000) # kind of a infinite patience to let max epochs rule
clf_bag = NeuralNet(**NNargs)
clf_bag.fit(X_all, y_all)
probs_bags = clf_bag.predict_proba(X_test)
return probs_bags
def classify(train_file, test_file, output_file):
test = pd.read_csv(test_file)
train = pd.read_csv(train_file)
test['age'] = test['age'] / train['age'].max()
train['age'] = train['age'] / train['age'].max()
train.replace(to_replace={'Transfer': 0, 'Return_to_owner': 1, 'Euthanasia': 2, 'Adoption': 3, 'Died': 4}, inplace=True)
test.replace(to_replace={'Transfer': 0, 'Return_to_owner': 1, 'Euthanasia': 2, 'Adoption': 3, 'Died': 4}, inplace=True)
train_X = train.ix[:, features].as_matrix().astype('float32')
train_Y = train.ix[:, 'outcome'].as_matrix().astype('int32')
test_X = test.ix[:, features].as_matrix().astype('float32')
model = NeuralNet(
layers=[
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape=(None, len(features)),
hidden1_num_units=100, hidden1_nonlinearity=sigmoid,
hidden2_num_units=100, hidden2_nonlinearity=rectify,
max_epochs=100,
output_nonlinearity=softmax,
output_num_units=5,
update_learning_rate=0.01,
).fit(train_X, train_Y)
all_proba = model.predict_proba(test_X).reshape(test_X.shape[0], 5)
result = pd.DataFrame(
data=all_proba,
columns=['Transfer', 'Return_to_owner', 'Euthanasia', 'Adoption', 'Died'],
index=test['id']
)
result.to_csv(output_file, index_label="ID", float_format='%.5f')
def train_autoencoder(train, train_y, test):
num_features = train.shape[1]
net = NeuralNet(
layers=[
('input', InputLayer),
('auto', AutoEncoder),
('output', DenseLayer),
],
input_shape=(None, 1, num_features),
auto_num_units = 1000,
auto_n_hidden = 10,
output_num_units=1000,
update_learning_rate=theano.shared(float32(0.03)),
update_momentum=theano.shared(float32(0.9)),
output_nonlinearity=nonlinearities.softmax,
regression=True,
max_epochs=3,
verbose=1,
)
net.fit(train, train_y)
with open('net.pickle', 'wb') as f:
pickle.dump(net, f, -1)
pred_auto = net.predict_proba(test)
return pred_auto
def nn_level2(train_x, train_y, test_x):
'''
neural net proba predict for level 2
'''
num_classes = len(np.unique(train_y))
num_features = train_x.shape[1]
layers0 = [('input', InputLayer),
('dropoutf', DropoutLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('dense1', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dropoutf_p=0.15,
dense0_num_units=1000,
dropout_p=0.25,
dense1_num_units=500,
dropout2_p=0.25,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=theano.shared(np.float32(0.01)),
# on_epoch_finished=[AdjustVariable('update_learning_rate', start=0.02, stop=0.016)],
max_epochs=18,
eval_size=0.2,
verbose=1,
)
net0.fit(train_x, train_y)
pred = net0.predict_proba(test_x).astype(np.float32)
return pred
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout0', DropoutLayer),
('dense1', DenseLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=100,
dropout0_p=0.1,
dense1_num_units=100,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.3,
update_momentum=0.8,
#objective_loss_function = binary_crossentropy,
train_split=TrainSplit(0.1),
verbose=1,
max_epochs=20)
net0.fit(X, y)
print('Prediction Complete')
preds = net0.predict_proba(X_test)[:, 1]
submission = pd.DataFrame(preds, index=ids, columns=['target'])
submission.to_csv('Neural_Network.csv')
predictions = []
gbm = xgb.train(params,xgb.DMatrix(result.iloc[train_index,:],y.iloc[train_index]),num_trees)
rf.fit(X_train[train_index,:],y[train_index])
ada.fit(X_train[train_index,:],y[train_index])
svr.fit(X_train[train_index,:],y[train_index])
knn.fit(X_train[train_index,:],y[train_index])
y_train = encoder.fit_transform(y[train_index]).astype(np.int32)
net0.fit(X_train[train_index,:],y_train)
predictions.append(svr.predict_proba(X_train[test_index])[:,1])
predictions.append(gbm.predict(xgb.DMatrix(result.iloc[test_index])))
predictions.append(ada.predict_proba(X_train[test_index])[:,1])
predictions.append(rf.predict_proba(X_train[test_index])[:,1])
predictions.append(knn.predict_proba(X_train[test_index])[:,1])
predictions.append(net0.predict_proba(X_train[test_index])[:,1])
netAccuracy.append(roc_auc_score(y.iloc[test_index],net0.predict_proba(X_train[test_index])[:,1]))
starting_values = [1./6]*len(predictions)
# train_eval_probs = 0.30*svr.predict_proba(X_train[test_index])[:,1] + 0.15*gbm.predict(xgb.DMatrix(result.iloc[test_index])) \
# + 0.15*ada.predict_proba(X_train[test_index])[:,1] + 0.35*rf.predict_proba(X_train[test_index])[:,1] \
# + 0.05*knn.predict_proba(X_train[test_index])[:,1]
res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons)
print('Ensamble Score: {best_score}'.format(best_score=res['fun']))
print('Best Weights: {weights}'.format(weights=res['x']))
weights.append(res['x'])
# aucList.append(roc_auc_score(y.iloc[test_index],train_eval_probs))
print "Deep Learning - CNN"
print sum(netAccuracy)/10.
def neuralNetwork(x_train, y_train, x_test, y_test):
global get_test
#use neural network to classifier
print "Neural Network Model Learning..."
num_train_case = x_train.shape[0]
#get random permutation index
index = np.random.permutation(num_train_case)
#random permute x_train and y_train
x_train = x_train[index, :]
y_train = y_train[index]
#encode y
encoder = LabelEncoder()
y_train = encoder.fit_transform(y_train).astype(np.int32)
#print encoder.classes_
#scale x_train
#scaler = StandardScaler()
#x_train = scaler.fit_transform(x_train)
#scale x_test
#x_test = scaler.transform(x_test.astype(np.float32))
num_classes = len(encoder.classes_)
num_features = x_train.shape[1]
#Train Neural Net
layers0 = [('input', InputLayer), ('dense0', DenseLayer),
('dropout', DropoutLayer), ('dense1', DenseLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=2000,
dropout_p=0.5,
dense1_num_units=200,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
eval_size=0.2,
verbose=1,
max_epochs=20)
'''
epochs = []
def on_epoch_finished(nn, train_history):
epochs[:] = train_history
if len(epochs) > 1:
raise StopIteration()
net0 = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
#('dropout2', DropoutLayer),
('output', DenseLayer),
],
input_shape=(None, num_features),
output_num_units=num_classes,
output_nonlinearity=softmax,
hidden1_num_units=200,
hidden2_num_units=200,
dropout_p=0.5,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=20,
on_epoch_finished=on_epoch_finished,
)
#net0 = clone(nn_def)
'''
#model learning
net0.fit(x_train, y_train)
#probability prediction
y_prob = net0.predict_proba(x_test)
if get_test == True:
#the data is from real test set
#output to file
output_result(y_prob)
else:
#the test set is split from the train set, compute the loss function value
#encode string label 'Class_1', 'Class_2',... to [0,1,...,8]
y_true = encoder.transform(y_test).astype(np.int32)
#compute the value for loss function
score = logloss_mc(y_true, y_prob)
print(
" -- Multiclass logloss on validation set: {:.5f}.".format(score))
weights.append(res['x'])
# aucList.append(roc_auc_score(y.iloc[test_index],train_eval_probs))
print "Deep Learning - CNN"
print sum(netAccuracy)/10.
#%%
resultNet = net0.predict_proba(X_test)
result = gbm.predict(xgb.DMatrix(X_test))
result_lb = lb.predict_proba(X_test)
submit = pd.read_csv('/Users/weizhi/Desktop/kaggle walmart competetion/sample_submission.csv')
Id = submit['VisitNumber']
submit.iloc[:,1:] = resultNet
submit.iloc[:,0] = Id
submit.to_csv('/Users/weizhi/Desktop/kaggle walmart competetion/deeplearning.csv',index = False)
}), (layers.DropoutLayer, {
'p': 0.5
}),
(layers.DenseLayer, {
'num_units': 10,
'nonlinearity': nonlinearities.softmax
})]
net = NeuralNet(
layers=layer,
update=updates.adam,
update_learning_rate=theano.shared(float32(0.002)),
verbose=1,
batch_iterator_train=batch_iterator_train,
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.002, stop=0.0005)
],
eval_size=0.0,
max_epochs=30,
)
print("Training neural network...")
net.fit(X, y)
del X
X_test, ids = load_test_data(path, grayscale=False, img_shape=IMG_SHAPE)
print("Predicting on test data...")
y_proba = net.predict_proba(X_test)
make_submission('../output/submission_01.csv', y_proba, ids)
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
max_epochs=50,
eval_size=None,
verbose=1,
)
X_train, y_train = shuffle(X_train, y_train, random_state=888)
net.fit(X_train, y_train)
y_pred = net.predict_proba(X_test)[:, 0]
score = metrics.roc_auc_score(y_test_pd, y_pred)
scores.append(score)
total_pred = np.concatenate((total_pred, y_pred))
total_test = np.concatenate((total_test, y_test_pd))
print("Global ROC score", metrics.roc_auc_score(total_test, total_pred))
print(scores)
print(np.array(scores).mean())
else:
clf.fit(X, train.WnvPresent)
# AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
# AdjustVariable('update_momentum', start=0.9, stop=0.999),
# ],
#on_epoch_finished=[
# AdjustVariable(learning_rate, target=0, half_life=4),
#],
eval_size=0.2,
verbose=1,
max_epochs=32,
batch_iterator_train=BatchIterator(batch_size=64),
batch_iterator_test=BatchIterator(batch_size=64))
net0.fit(X, y)
X_train, X_valid, y_train, y_valid = net0.train_test_split(
X, y, net0.eval_size)
probas = net0.predict_proba(X_valid)[:, 0]
#probas.loc[(probas[0]<0.3).values,[0]] = 0.1
print("ROC score", metrics.roc_auc_score(y_valid, (probas)))
#yp = DataFrame(net0.predict_proba(X_test),columns=[ u'Class_1', u'Class_2', u'Class_3', u'Class_4', u'Class_5', u'Class_6', u'Class_7', u'Class_8', u'Class_9'])
pred_prob = pd.DataFrame(net0.predict(X_test))
print pred_prob
sample = pd.read_csv('/Users/IkkiTanaka/Documents/KDDCup/sampleSubmission.csv',
header=None)
preds = pd.concat([sample[0], pd.DataFrame(pred_prob)], axis=1)
preds.to_csv('/Users/IkkiTanaka/Documents/KDDCup/pred/xgb/predNN1.csv',
header=None,
index=False)
for i in xrange(0, 50):
layers0 = [
# Decay the learning rate
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.99),
],
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
#batch_iterator_train = BatchIterator(batch_size = 256),
max_epochs=20,
eval_size=0.1,
#train_split =0.0,
verbose=2,
)
seednumber = 1235
np.random.seed(seednumber)
net.fit(train, labels)
preds = net.predict_proba(test)[:, 0]
submission = pd.read_csv('../input/sample_submission.csv')
submission["PredictedProb"] = preds
submission.to_csv('NNbench.csv', index=False)
class NNnolearn:
packageName = "com.brodagroup.machinelearning.classifer.NNnolearn"
logger = None
classifier = None
# Initializer
def __init__(
self,
num_classes=None,
num_features=None,
dense0_num_units=10,
dropout_p=0.1,
dense1_num_units=10,
update_learning_rate=0.1,
update_momentum=0.1,
eval_size=0.1,
max_epochs=10,
verbose=5,
):
self.logger = Logger(self.packageName).getLogger()
self.logger.debug("Starting...")
layers0 = [
("input", InputLayer),
("dense0", DenseLayer),
("dropout1", DropoutLayer),
("dense1", DenseLayer),
("output", DenseLayer),
]
self.classifier = NeuralNet(
layers=layers0,
input_shape=(None, num_features),
dense0_num_units=dense0_num_units,
dropout1_p=dropout_p,
dense1_num_units=dense1_num_units,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=update_learning_rate,
update_momentum=update_momentum,
eval_size=eval_size,
max_epochs=max_epochs,
verbose=verbose,
)
self.num_classes = num_classes
self.num_features = num_features
self.dense0_num_units = dense0_num_units
self.dropout_p = dropout_p
self.dense1_num_units = dense1_num_units
self.update_learning_rate = update_learning_rate
self.update_momentum = update_momentum
self.eval_size = eval_size
self.max_epochs = max_epochs
self.verbose = verbose
return
def __str__(self):
return x
def fit(self, X, y):
self.classifier.fit(X, y)
def predict(self, X):
y_pred = self.classifier.predict(X)
return y_pred
def predict_proba(self, X):
y_pred = self.classifier.predict_proba(X)
return y_pred
def get_params(self, deep=True):
return {
"num_classes": self.num_classes,
"num_features": self.num_features,
"dense0_num_units": self.dense0_num_units,
"dropout_p": self.dropout_p,
"dense1_num_units": self.dense1_num_units,
"update_learning_rate": self.update_learning_rate,
"eval_size": self.eval_size,
"max_epochs": self.max_epochs,
"verbose": self.verbose,
}
def set_params(self, **parameters):
for parameter, value in parameters.items():
setattr(self, parameter, value)
return self
objective_loss_function=categorical_crossentropy,
train_split=TrainSplit(eval_size=0),
verbose=1,
max_epochs=1)
# ==== Print out input shape for diagnosis ====
print(X.shape)
print(training_target.shape)
# ==== Train it for n iterations and validate on each iteration ====
spl = 80000
for i in range(epochs):
net1.fit(x2[:spl, :], training_target[:spl])
print(i + 1)
pred = net1.predict_proba(x2[spl:, :])[:, 1]
val_auc[i] = roc_auc_score(training_target[spl:], pred)
print(i + 1, "\t", round(val_auc[i] * 100, 3), "\t",
round(max(val_auc) * 100, 3), "\t")
pred_d = pd.read_csv(path_pred)
del pred_d["t_id"]
for f in features:
for g in features:
if f != g:
if not (str(g) + "_" + str(f)) in pred_d.columns:
pred_d[str(f) + "_" + str(g)] = pred_d[f] * pred_d[g]
X = pred_d.values.astype(np.float32)
X = scaler.transform(X).astype(np.float32)
proba = net1.predict_proba(y2)[:, 1]
dropout_p=0.5,
dense1_num_units=200,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
eval_size=0.2,
verbose=1,
max_epochs=200)
print "fitting nn model.."
net0.fit(X_train, y_train)
print "predicting probabilities using nn model..."
proba = net0.predict_proba(X_test)
ll.append(calc_ll_from_proba(proba, y_test))
print metrics.confusion_matrix(
y_test.astype(int), np.argmax(proba, axis=1).astype(int))
print "logloss: ", ll[ncv]
print "saving nn model..."
net0.save_weights_to('weights/nn_%d.pkl' % ncv)
net0 = None
ncv += 1
ll = np.array(ll)
print "logloss: ", ll
dropout3_p=0.3,
hidden4_num_units=230,
hidden4_nonlinearity=very_leaky_rectify,
dropout4_p=0.5,
hidden5_num_units=150,
dropout5_p=0.35,
output_num_units=2, output_nonlinearity=softmax,
update_learning_rate=theano.shared(float32(0.02)),
update_momentum=theano.shared(float32(0.9)),
# regression=True,
batch_iterator_train=FlipBatchIterator(batch_size=128),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.02, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=80),
],
max_epochs=2000,
verbose=1,
train_split=TrainSplit(eval_size=0.2),
)
if __name__ == '__main__':
X, y , Xtest= load2d() # load 2-d data
net.fit(X.astype(theano.config.floatX), y.astype(np.int32).reshape(-1,))
ypred = net.predict_proba(Xtest.astype(theano.config.floatX))
with open('net3.res.pickle', 'wb') as f:
pickle.dump(ypred, f)
with open('net3.pickle', 'wb') as f:
pickle.dump(net,f)
objective_loss_function=objectives.categorical_crossentropy,
#objective=objectives.categorical_crossentropy,
batch_iterator_train=BatchIterator(batch_size=batchSize),
batch_iterator_test=BatchIterator(batch_size=batchSize),
train_split=TrainSplit(eval_size=0.2, stratify=False),
use_label_encoder=True,
#use_label_encoder = False,
regression=False,
max_epochs=numberEpochs,
verbose=1)
#x_fit, x_eval, y_fit, y_eval= cross_validation.train_test_split(xTrain, y, test_size=0.2)
net.fit(xTrain, y)
predictY = net.predict_proba(xTrain)
print(metrics.log_loss(originalY, predictY))
files = [f for f in listdir(testDir) if path.isfile(path.join(testDir, f))]
xTest = np.zeros((len(files), imageSize), dtype='float32')
index = 0
for fName in files:
fileName = path.join(testDir, fName)
im = Image.open(fileName)
xTest[index, :] = np.reshape(im, imageSize)
im.close()
index += 1
xTest /= 255
xTest = xTest.reshape(xTest.shape[0], 1, imageShape[0],
imageShape[1]).astype('float32')
update_learning_rate=theano.shared(np.float32(0.013)),
###
regression=False,
max_epochs=2000,
train_split=TrainSplit(eval_size=0.1),
#custom_score=('auc', lambda y_true, y_proba: roc_auc_score(y_true, y_proba[:, 1])),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.013, stop=0.001),
#AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=60),
],
verbose=1)
clf.fit(X_train, y_train)
from sklearn import metrics
y_pred = clf.predict_proba(X_val0)[:, 1]
score = metrics.roc_auc_score(y_val0, y_pred)
print('score on extra set:%s' % score)
model = 'Nolearn'
#
# predict on test set
submission = '%s_score_%03f.csv' % (model, score)
# create submission file
predictions = clf.predict_proba(X_test)[:, 1]
preds = pd.DataFrame({"patient_id": id_test, 'predict_screener': predictions})
test_ex_file = ('../input/test_patients_to_exclude.csv.gz')
test_ex = pd.read_csv(test_ex_file, low_memory=False)
random_state=1)
print("\n\nTraining")
net = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape=(None, 6),
hidden_num_units=5,
output_nonlinearity=sigmoid,
output_num_units=2,
# optimization method:
update=sgd,
update_learning_rate=0.01,
#update_momentum=0.9,
regression=False,
max_epochs=50,
verbose=1,
)
net.fit(X, y)
pred = net.predict_proba(tX)[:,0]
print("\t", log_loss(ty, pred))
print("\t", roc_auc_score(ty, pred))
class NNet(BaseEstimator, ClassifierMixin):
def __init__(
self,
name='nameless_net', # used for saving, so maybe make it unique
dense1_size=60,
dense1_nonlinearity='tanh',
dense1_init='orthogonal',
dense2_size=None,
dense2_nonlinearity=None, # inherits dense1
dense2_init=None, # inherits dense1
dense3_size=None,
dense3_nonlinearity=None, # inherits dense2
dense3_init=None, # inherits dense2
learning_rate=0.001,
learning_rate_scaling=100,
momentum=0.9,
momentum_scaling=100,
max_epochs=3000,
epoch_steps=None,
dropout0_rate=0, # this is the input layer
dropout1_rate=None,
dropout2_rate=None, # inherits dropout1_rate
dropout3_rate=None, # inherits dropout2_rate
weight_decay=0,
adaptive_weight_decay=False,
batch_size=128,
output_nonlinearity='softmax',
auto_stopping=True,
save_snapshots_stepsize=None,
):
"""
Create the network with the selected parameters.
:param name: Name for save files
:param dense1_size: Number of neurons for first hidden layer
:param dense1_nonlinearity: The activation function for the first hidden layer
:param dense1_init: The weight initialization for the first hidden layer
:param learning_rate: The (initial) learning rate (how fast the network learns)
:param learning_rate_scaling: The total factor to gradually decrease the learning rate by
:param momentum: The (initial) momentum
:param momentum_scaling: Similar to learning_rate_scaling
:param max_epochs: Total number of epochs (at most)
:param dropout1_rate: Percentage of connections dropped each step for first hidden layer
:param weight_decay: Palatalizes the weights by L2 norm (regularizes but decreases results)
:param adaptive_weight_decay: Should the weight decay adapt automatically?
:param batch_size: How many samples to send through the network at a time
:param auto_stopping: Stop early if the network seems to stop performing well
:param pretrain: Filepath of the previous weights to start at (or None)
:return:
"""
"""
Input argument storage: automatically store all locals, which should be exactly the arguments at this point, but storing a little too much is not a big problem.
"""
params = locals()
del params['self']
#self.__dict__.update(params)
self.parameter_names = sorted(params.keys())
"""
Check the parameters and update some defaults (will be done for 'self', no need to store again).
"""
self.set_params(**params)
def init_net(self,
feature_count,
class_count=NCLASSES,
verbosity=VERBOSITY >= 2):
"""
Initialize the network (needs to be done when data is available in order to set dimensions).
"""
if VERBOSITY >= 1:
print 'initializing network {0:s} {1:d}x{2:d}x{3:d}'.format(
self.name, self.dense1_size or 0, self.dense2_size or 0,
self.dense3_size or 0)
if VERBOSITY >= 2:
print 'parameters: ' + ', '.join(
'{0:s} = {1:}'.format(k, v)
for k, v in self.get_params(deep=False).items())
self.feature_count = feature_count
self.class_count = class_count
"""
Create the layers and their settings.
"""
self.layers = [
('input', InputLayer),
]
self.params = {
'dense1_num_units': self.dense1_size,
'dense1_nonlinearity': nonlinearities[self.dense1_nonlinearity],
'dense1_W': initializers[self.dense1_init],
'dense1_b': Constant(0.),
}
if self.dropout0_rate:
self.layers += [('dropout0', DropoutLayer)]
self.params['dropout0_p'] = self.dropout0_rate
self.layers += [
('dense1', DenseLayer),
]
if self.dropout1_rate:
self.layers += [('dropout1', DropoutLayer)]
self.params['dropout1_p'] = self.dropout1_rate
if self.dense2_size:
self.layers += [('dense2', DenseLayer)]
self.params.update({
'dense2_num_units':
self.dense2_size,
'dense2_nonlinearity':
nonlinearities[self.dense2_nonlinearity],
'dense2_W':
initializers[self.dense2_init],
'dense2_b':
Constant(0.),
})
else:
assert not self.dense3_size, 'There cannot be a third dense layer without a second one'
if self.dropout2_rate:
assert self.dense2_size is not None, 'There cannot be a second dropout layer without a second dense layer.'
self.layers += [('dropout2', DropoutLayer)]
self.params['dropout2_p'] = self.dropout2_rate
if self.dense3_size:
self.layers += [('dense3', DenseLayer)]
self.params.update({
'dense3_num_units':
self.dense3_size,
'dense3_nonlinearity':
nonlinearities[self.dense3_nonlinearity],
'dense3_W':
initializers[self.dense3_init],
'dense3_b':
Constant(0.),
})
if self.dropout3_rate:
assert self.dense2_size is not None, 'There cannot be a third dropout layer without a third dense layer.'
self.layers += [('dropout3', DropoutLayer)]
self.params['dropout3_p'] = self.dropout2_rate
self.layers += [('output', DenseLayer)]
self.params.update({
'output_nonlinearity':
nonlinearities[self.output_nonlinearity],
'output_W':
GlorotUniform(),
'output_b':
Constant(0.),
})
"""
Create meta parameters and special handlers.
"""
if VERBOSITY >= 3:
print 'learning rate: {0:.6f} -> {1:.6f}'.format(
abs(self.learning_rate),
abs(self.learning_rate) / float(self.learning_rate_scaling))
print 'momentum: {0:.6f} -> {1:.6f}'.format(
abs(self.momentum),
1 - ((1 - abs(self.momentum)) / float(self.momentum_scaling)))
self.step_handlers = [
LinearVariable('update_learning_rate',
start=abs(self.learning_rate),
stop=abs(self.learning_rate) /
float(self.learning_rate_scaling)),
LinearVariable(
'update_momentum',
start=abs(self.momentum),
stop=1 -
((1 - abs(self.momentum)) / float(self.momentum_scaling))),
StopNaN(),
]
self.end_handlers = [
SnapshotEndSaver(base_name=self.name),
TrainProgressPlotter(base_name=self.name),
]
snapshot_name = 'nn_' + params_name(self.params, prefix=self.name)[0]
if self.save_snapshots_stepsize:
self.step_handlers += [
SnapshotStepSaver(every=self.save_snapshots_stepsize,
base_name=snapshot_name),
]
if self.auto_stopping:
self.step_handlers += [
StopWhenOverfitting(loss_fraction=0.9,
base_name=snapshot_name),
StopAfterMinimum(patience=40, base_name=self.name),
]
weight_decay = shared(float32(abs(self.weight_decay)), 'weight_decay')
if self.adaptive_weight_decay:
self.step_handlers += [
AdaptiveWeightDecay(weight_decay),
]
if self.epoch_steps:
self.step_handlers += [
BreakEveryN(self.epoch_steps),
]
"""
Create the actual nolearn network with information from __init__.
"""
self.net = NeuralNet(
layers=self.layers,
objective=partial(WeightDecayObjective, weight_decay=weight_decay),
input_shape=(None, feature_count),
output_num_units=class_count,
update=nesterov_momentum, # todo: make parameter
update_learning_rate=shared(float32(self.learning_rate)),
update_momentum=shared(float(self.weight_decay)),
on_epoch_finished=self.step_handlers,
on_training_finished=self.end_handlers,
regression=False,
max_epochs=self.max_epochs,
verbose=verbosity,
batch_iterator_train=BatchIterator(batch_size=self.batch_size),
batch_iterator_test=BatchIterator(batch_size=self.batch_size),
eval_size=0.1,
#custom_score = ('custom_loss', categorical_crossentropy),
**self.params)
self.net.parent = self
self.net.initialize()
return self.net
def get_params(self, deep=True):
return OrderedDict(
(name, getattr(self, name)) for name in self.parameter_names)
def set_params(self, **params):
"""
Set all the parameters.
"""
for name, val in params.items():
assert name in self.parameter_names, '"{0:s}" is not a valid parameter name (known parameters: "{1:s}")'.format(
name, '", "'.join(self.parameter_names))
setattr(self, name, val)
"""
Arguments checks.
"""
assert self.dropout1_rate is None or 0 <= self.dropout1_rate < 1, 'Dropout rate 1 should be a value between 0 and 1 (value: {0})'.format(
self.dropout1_rate)
assert self.dropout2_rate is None or 0 <= self.dropout2_rate < 1, 'Dropout rate 2 should be a value between 0 and 1, or None for inheritance (value: {0})'.format(
self.dropout2_rate)
assert self.dropout3_rate is None or 0 <= self.dropout3_rate < 1, 'Dropout rate 3 should be a value between 0 and 1, or None for inheritance (value: {0})'.format(
self.dropout3_rate)
assert self.dense1_nonlinearity in nonlinearities.keys(
), 'Linearity 1 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(nonlinearities.keys()), self.dense1_nonlinearity)
assert self.dense2_nonlinearity in nonlinearities.keys() + [
None
], 'Linearity 2 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(nonlinearities.keys()), self.dense2_nonlinearity)
assert self.dense3_nonlinearity in nonlinearities.keys() + [
None
], 'Linearity 3 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(nonlinearities.keys()), self.dense3_nonlinearity)
assert self.dense1_init in initializers.keys(
), 'Initializer 1 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(initializers.keys()), self.dense1_init)
assert self.dense2_init in initializers.keys() + [
None
], 'Initializer 2 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(initializers.keys()), self.dense2_init)
assert self.dense3_init in initializers.keys() + [
None
], 'Initializer 3 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(initializers.keys()), self.dense3_init)
"""
Argument defaults.
"""
if self.dense2_nonlinearity is None:
self.dense2_nonlinearity = self.dense1_nonlinearity
if self.dense2_init is None:
self.dense2_init = self.dense1_init
if self.dense3_nonlinearity is None:
self.dense3_nonlinearity = self.dense2_nonlinearity
if self.dense3_init is None:
self.dense3_init = self.dense2_init
if self.dropout2_rate is None and self.dense2_size:
self.dropout2_rate = self.dropout1_rate
if self.dropout3_rate is None and self.dense3_size:
self.dropout3_rate = self.dropout2_rate
def fit(self, X, y, random_sleep=None):
if random_sleep:
sleep(random_sleep *
random()) # this is to prevent compiler lock problems
labels = y - y.min()
#todo: don't use labels.max(), occasionally (rarely) it will not have the highest class
self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1)
net = self.net.fit(X, labels)
self.save()
return net
def interrupted_fit(self, X, y):
""" DEPRECATED """
labels = y - y.min()
self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1)
knowledge = get_knowledge(self.net)
for epoch in range(0, self.max_epochs, self.epoch_steps):
set_knowledge(self.net, knowledge)
self.init_net(feature_count=X.shape[1],
class_count=labels.max() + 1)
print 'epoch {0:d}: learning {1:d} epochs'.format(
epoch, self.epoch_steps)
self.net.fit(X, labels)
ratio = mean([d['valid_loss'] for d in self.net._train_history[-self.epoch_steps:]]) / \
mean([d['train_loss'] for d in self.net._train_history[-self.epoch_steps:]])
if ratio < 0.85:
self.weight_decay *= 1.3
if ratio > 0.95:
self.weight_decay /= 1.2
self.init_net(feature_count=X.shape[1],
class_count=labels.max() + 1)
knowledge = get_knowledge(self.net)
exit()
net = self.net.fit(X, labels)
self.save()
return net
def predict_proba(self, X):
probs = self.net.predict_proba(X)
if not isfinite(probs).sum():
errmsg = 'network "{0:s}" predicted infinite/NaN probabilities'.format(
self.name)
stderr.write(errmsg)
raise DivergenceError(errmsg)
return probs
def predict(self, X):
return self.net.predict(X)
def score(self, X, y, **kwargs):
return self.net.score(X, y)
def save(self, filepath=None):
assert hasattr(
self, 'net'
), 'Cannot save a network that is not initialized; .fit(X, y) something first [or use net.initialize(..) for random initialization].'
parameters = self.get_params(deep=False)
filepath = filepath or join(NNET_STATE_DIR, self.name)
if VERBOSITY >= 1:
print 'saving network to "{0:s}.net.npz|json"'.format(filepath)
with open(filepath + '.net.json', 'w+') as fh:
dump([parameters, self.feature_count, self.class_count],
fp=fh,
indent=2)
save_knowledge(self.net, filepath + '.net.npz')
@classmethod
def load(cls, filepath=None, name=None):
"""
:param filepath: The base path (without extension) to load the file from, OR:
:param name: The name of the network to load (if filename is not given)
:return: The loaded network
"""
filepath = filepath or join(NNET_STATE_DIR, name)
if VERBOSITY >= 1:
print 'loading network from "{0:s}.net.npz|json"'.format(filepath)
with open(filepath + '.net.json', 'r') as fh:
[parameters, feature_count, class_count] = load(fp=fh)
nnet = cls(**parameters)
nnet.init_net(feature_count=feature_count, class_count=class_count)
load_knowledge(nnet.net, filepath + '.net.npz')
return nnet
AdjustVariable('update_momentum', start=0.9, stop=0.999),
],
regression=False, # flag to indicate we're dealing with regression problem
max_epochs=500, # we want to train this many epochs
verbose=1
)
clf.fit(x_train,y_train)
# if 1:
# y_pred = clf.predict_proba(x_test)
#
# filename = 'testdata_aug_d'
# savefile = open(filename+'.pkl', 'wb')
# cPickle.dump((x_test, y_pred, name1),savefile,-1)
# savefile.close()
if 1:
y_pred = clf.predict(x_test)
print "Accuracy:", zero_one_loss(y_test, y_pred)
print "Classification report:"
print classification_report(y_test, y_pred)
print 'Confusion matrix:'
print confusion_matrix(y_test,y_pred)
else:
x_test = np.asarray(x_test,dtype=np.float32)
ypred = clf.predict_proba(x_test)
y_str = ['Class_1','Class_2','Class_3','Class_4','Class_5','Class_6','Class_7','Class_8','Class_9']
kcsv.print_csv(ypred, name1, y_str,indexname='id')
def do_nn(xTrain, yTrain, test_x=None, test_y=None, dropout_in=0.2, dense0_num=600, dropout_p=0.4, dense1_num=1200,
update_learning_rate=0.00002,
update_momentum=0.9, test_ratio=0.2, max_epochs=40, search=False):
num_features = len(xTrain[0, :])
num_classes = 2
print num_features
if search == False:
layers0 = [('input', InputLayer),
('dropoutin', DropoutLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('dense1', DenseLayer),
('output', DenseLayer)]
clf = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dropoutin_p=dropout_in,
dense0_num_units=dense0_num,
dropout_p=dropout_p,
dense1_num_units=dense1_num,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=update_learning_rate,
update_momentum=update_momentum,
eval_size=test_ratio,
verbose=1,
max_epochs=max_epochs)
clf.fit(xTrain, yTrain)
if test_x != None and test_y != None:
probas_nn = clf.predict_proba(test_x)[:, 1]
score_nn = roc_auc_score(test_y, probas_nn)
print("NN ROC score", score_nn)
return clf
else:
dropout_in_list = [0.2]
dense0_num_list = [500, 1000, 1500]
dropout_p_list = [0.5]
dense1_num_list = [400, 800, 1200]
info = {}
for d_in in dropout_in_list:
for d_01 in dropout_p_list:
for d0 in dense0_num_list:
for d1 in dense1_num_list:
print 'dropout_in = ', d_in
print 'dense0_num = ', d0
print 'dropout_p = ', d_01
print 'dense0_num = ', d1
layers0 = [('input', InputLayer),
('dropoutin', DropoutLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('dense1', DenseLayer),
('output', DenseLayer)]
clf = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dropoutin_p=dropout_in,
dense0_num_units=dense0_num,
dropout_p=dropout_p,
dense1_num_units=dense1_num,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=update_learning_rate,
update_momentum=update_momentum,
eval_size=test_ratio,
verbose=1,
max_epochs=max_epochs)
clf.fit(xTrain, yTrain)
probas_nn = clf.predict_proba(test_x)[:, 1]
score_nn = roc_auc_score(test_y, probas_nn)
print("NN ROC score", score_nn)
info[d_in, d0, d_01, d1] = score_nn
for md in info:
scores = info[md]
print('NN dropout_in = %.5f, dense0_num = %d, dropout_p = %.5f, dense1_num = %d, ROC score = %.5f(%.5f)' % \
(md[0], md[1], md[2], md[3], scores.mean(), scores.std()))
def make_predictions(train, target, test, ranking=False, load_list=[]):
result = np.zeros(len(test))
seed_list = [1234, 2345, 3456, 4567, 5678, 6789, 7890, 8901, 9012, 123]
print("Training model1...")
if "linear" not in load_list:
model1 = sklearn.linear_model.SGDClassifier(loss="log", alpha=0.01, l1_ratio=0, n_iter=100)
model1.fit(train, target)
pickle.dump(model1, open("final_models/linear/lr.pkl", "wb"))
else:
model1 = pickle.load(open("final_models/linear/lr.pkl", "rb"))
pred1 = model1.predict_proba(test)[:, 1]
print("Training model2...")
pred2 = np.zeros(len(pred1))
for i in range(10):
if "xgb" not in load_list:
model2 = xgb.XGBClassifier(n_estimators=500, max_depth=4, colsample_bytree=0.6,
subsample=0.8, learning_rate=0.09, seed=seed_list[i])
model2.fit(train, target)
pickle.dump(model2, open("final_models/xgb/xgb_n_"+str(i)+".pkl", "wb"))
else:
model2 = pickle.load(open("final_models/xgb/xgb_n_"+str(i)+".pkl", "rb"))
pred2 += model2.predict_proba(test)[:, 1]
pred2 /= 10
print("Training model3...")
pred3 = np.zeros(len(pred2))
for i in range(10):
if "nn" not in load_list:
np.random.seed(seed_list[i])
num_classes = 2
layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout1', DropoutLayer),
('dense1', DenseLayer), ('dropout2', DropoutLayer),
('dense2', DenseLayer), ('output', DenseLayer)]
model3 = NeuralNet(layers=layers0, input_shape=(None, train.shape[1]), dense0_num_units=150,
dropout1_p=0.4, dense1_num_units=150,
dropout2_p=0.4, dense2_num_units=150,
output_num_units=num_classes,
output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.001,
update_momentum=0.9, eval_size=0.01, verbose=0,
max_epochs=100, use_label_encoder=True)
model3.fit(train, target)
pickle.dump(model3, open("final_models/nn/nn_n_"+str(i)+".pkl", "wb"))
else:
model3 = pickle.load(open("final_models/nn/nn_n_"+str(i)+".pkl", "rb"))
pred3 += model3.predict_proba(test)[:, 1]
pred3 /= 10
if ranking:
pred1 = scipy.stats.rankdata(pred1)
pred2 = scipy.stats.rankdata(pred2)
pred3 = scipy.stats.rankdata(pred3)
result = 0.21*pred1 + 0.47*pred2 + 0.32*pred3
return result
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
mean, std = normalize(X)
y = assemble_y(training)
col_labels = ["date.year","date.month","date.day","block","lat","long",]
for obs in ["Tmax","Tmin","Tavg","DewPoint","WetBulb","PrecipTotal","Depart","Sunrise","Sunset","Heat","Cool","ResultSpeed","ResultDir"]:
for day in ["1","2","3","5","8","12"]:
col_labels.append(obs+"_"+day)
for i_spec in range(6):
col_labels.append("species_"+str(i_spec))
X_file = csv.writer(open("X.csv", "w"))
X_file.writerow(col_labels)
for row in X:
X_file.writerow(row)
y_file = csv.writer(open("y.csv", "w"))
y_file.writerow(["y"])
for row in y:
y_file.writerow(row)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=600,
dropout1_p=0.5,
hidden2_num_units=400,
dropout2_p=0.5,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=20),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=500,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
# pre-processing data
df.target = df.target.apply(lambda x: x.split("_", 1)[1]) # convert class_x -> x
df.target = df.target.astype(np.int32)
df.target = df.target - 1
df = df.iloc[:,1:]
df = df.reindex(np.random.permutation(df.index))
df_test = df_test.iloc[:,1:] # take out id col
X_test = df_test.values.astype(np.float32)
X, y = df.values[:,:93].astype(np.float32), df.values[:,93].astype(np.int32)
# training
result = myNN.fit(X,y)
# predict and prepare submission file
y_prob = myNN.predict_proba(X_test)
y_prob_df = pd.DataFrame(y_prob)
id_df = pd.DataFrame({'id':range(1,len(df_test)+1)})
final_df = pd.concat([id_df, y_prob_df], axis=1)
final_df.columns = ['id', 'Class_1', 'Class_2', 'Class_3', 'Class_4', 'Class_5', 'Class_6', 'Class_7', 'Class_8', 'Class_9']
final_df.to_csv('final_result.csv', index = False)
def main():
# my code here
y_train=load_train()
y_test =load_test()
X_train=load_images("/home/pratik/Desktop/Skinzy Code SVM/Datasets2/scabies_train.gz")
X_test=load_images("/home/pratik/Desktop/Skinzy Code SVM/Datasets2/scabies_test.gz")
X_train,y_train= shuffle(X_train, y_train, random_state=0)
X_val=X_train[80:]
X_train=X_train[:80]
y_val=y_train[80:]
y_train=y_train[:80]
y_train =np.array(y_train)
y_test=np.array(y_test)
y_val=np.array(y_val)
y_test = y_test.astype(np.uint8)
y_train = y_train.astype(np.uint8)
y_val = y_val.astype(np.uint8)
X_train,y_train= shuffle(X_train, y_train, random_state=0)
#print("Plotting the graph")
#plt.imshow(X_train[0][0])
net1 = NeuralNet(
layers=[('input', layers.InputLayer),
('conv2d1', layers.Conv2DLayer),
('maxpool1', layers.MaxPool2DLayer),
('conv2d2', layers.Conv2DLayer),
('maxpool2', layers.MaxPool2DLayer),
('dropout1', layers.DropoutLayer),
('dense', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('output', layers.DenseLayer),
],
# input layer
input_shape=(None, 3, 64,64),
# layer conv2d1
conv2d1_num_filters=32,
conv2d1_filter_size=(5, 5),
conv2d1_nonlinearity=lasagne.nonlinearities.rectify,
conv2d1_W=lasagne.init.GlorotUniform(),
# layer maxpool1
maxpool1_pool_size=(3, 3),
maxpool1_stride=1,
maxpool1_pad=0,
# layer conv2d2
conv2d2_num_filters=32,
conv2d2_filter_size=(5, 5),
conv2d2_nonlinearity=lasagne.nonlinearities.rectify,
# layer maxpool2
maxpool2_pool_size=(3, 3),
maxpool2_stride=1,
maxpool2_pad=0,
# dropout1
dropout1_p=0.5,
# dense
dense_num_units=256,
dense_nonlinearity=lasagne.nonlinearities.rectify,
# dropout2
dropout2_p=0.5,
# output
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=2,
# optimization method params
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.9,
max_epochs=50,
verbose=1,
)
print ("Training starts :")
net1.fit(X_train, y_train)
#preds = net1.predict(X_test[0])
out = X_train[0].reshape(-1, 3, 64, 64)
pred=net1.predict_proba(out)
print pred
#cm = confusion_matrix(y_test, preds)
#plt.matshow(cm)
#plt.title('Confusion matrix')
#plt.colorbar()
#plt.ylabel('True label')
#plt.xlabel('Predicted label')
#plt.show()
#print (net1.predict_proba(X_test))
sys.setrecursionlimit(9999999)
joblib.dump(net1, 'AndroidFileUpload/classifier_2disease/cnn_1.pkl',compress=9)
y_true, y_pred = y_test, net1.predict(X_test) # Get our predictions
print(classification_report(y_true, y_pred)) # Classification on each digit
print 'The accuracy is:', accuracy_score(y_true, y_pred)
# visualize.plot_conv_weights(net1.layers_['conv2d1'])
dense_layer = layers.get_output(net1.layers_['dense'], deterministic=True)
output_layer = layers.get_output(net1.layers_['output'], deterministic=True)
input_var = net1.layers_['input'].input_var
f_output = theano.function([input_var], output_layer)
f_dense = theano.function([input_var], dense_layer)
instance = X_test[0][None, :, :]
#%timeit -n 500 f_output(instance)
train_features = f_dense(X_train)
test_features=f_dense(X_test)
train_labels=y_train
test_labels=y_test
'''
Logistic Regression
clf2 = LogisticRegression().fit(train_features, train_labels)
joblib.dump(clf2, 'AndroidFileUpload/classifier_2disease/log_reg.pkl',compress=9)'''
#SVM
clf = svm.SVC(kernel="linear",C=10000.0,probability=True)
clf.fit(train_features,train_labels)
sys.setrecursionlimit(9999999)
joblib.dump(clf, 'AndroidFileUpload/classifier_2disease/svm.pkl',compress=9)
pred=clf.predict(test_features)
print(classification_report(test_labels, pred))
print 'The accuracy is:', accuracy_score(test_labels, pred)
pred=clf.predict_proba(test_features)
print pred
'''#KNN
output_num_units=4,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.899,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=4),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=75,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(Xtrh, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
for dy in range(-6, 6):
if nuclei_map[limit(i + dx, 0, SIZE - 1),
limit(j + dy, 0, SIZE - 1)]:
cover = True
for xx, yy in coords:
if i == xx and j == yy:
print("Found coordinate of mitosis: ", (xx, yy))
print("Cover = ", cover)
if cover:
patches.append((i, j))
if len(patches) >= BUFFER:
print("Evaluating!")
patches2 = get_patches(patches)
prob = nn.predict_proba(patches2)
for k in range(0, len(patches)):
sx, sy = patches[k]
patch_probs[sx, sy] = prob[k, 1]
for xx, yy in coords:
if sx == xx and sy == yy:
print("Found coordinate of mitosis: ",
(xx, yy))
print("Prob = ", prob[k, 1])
patches = []
with suppress_stdout():
nn.load_params_from(netfile)
if len(patches) >= 1:
print("Evaluating!")
# In[2]:
import pickle
pickle.dump(net0, open('lasagne_model.pkl', 'wb'))
# In[106]:
print 'score', net0.score(features_scaled, labels)
# In[111]:
print 'logloss', log_loss(labels, net0.predict_proba(features_scaled))
# In[16]:
test_features, _ = get_features(test)
scaler.fit(test_features)
test_features = scaler.transform(test_features)
# In[18]:
proba = net0.predict_proba(test_features)
prediction = pd.DataFrame(proba, columns=label_encoder.classes_)
# ==== Neural network definition ====
net1 = NeuralNet(layers=l_out,
update=adadelta, update_rho=0.95, update_learning_rate=1.0,
objective_loss_function=categorical_crossentropy,
train_split=TrainSplit(eval_size=0), verbose=0, max_epochs=1)
# ==== Print out input shape for diagnosis ====
print(X.shape)
print(training_target.shape)
# ==== Train it for n iterations and validate on each iteration ====
for i in range(epochs):
net1.fit(X, training_target)
pred = net1.predict_proba(Y)[:, 1]
val_auc[i] = roc_auc_score(validation_target, pred)
print(i + 1, "\t", round(val_auc[i] * 100, 3), "\t", round(max(val_auc) * 100, 3), "\t")
# ==== Make of the plot of the validation accuracy per iteration ====
pyplot.plot(val_auc, linewidth=2)
pyplot.axhline(y=max(val_auc), xmin=0, xmax=epochs, c="blue", linewidth=3, zorder=0)
pyplot.grid()
pyplot.title("Maximum AUC is " + str(round(max(val_auc) * 100, 3)) + '%')
pyplot.xlabel("Epoch")
pyplot.ylabel("Validation AUC")
pyplot.show()
print("Fitting Sample 0")
net0.fit(X, y)
for i in range(1, 14):
print("Loading Sample " + str(i))
X, y, encoder, scaler1 = load_train_data(
datapath + "train_fixed_data.csv", i)
print("Fitting Sample " + str(i))
net0.fit(X, y)
print("Fitting Complete")
test_splits = 14
print("Loading Test Sample 0")
X_test, ids = load_test_data(datapath + "test_fixed_data.csv", scaler, 0)
print("Predicting Test Sample 0")
preds = net0.predict_proba(X_test)[:, 1]
submission = pd.DataFrame(preds, index=ids, columns=['target'])
submission.to_csv('BTB_Lasagne.csv')
for i in range(1, test_splits + 1):
print("Loading Test Sample " + str(i))
X_test, ids = load_test_data(datapath + "test_fixed_data.csv", scaler,
i)
print("Predicting Test Sample " + str(i))
preds = net0.predict_proba(X_test)[:, 1]
submission = pd.DataFrame(preds, index=ids, columns=['target'])
with open('BTB_Lasagne.csv', 'a') as f:
submission.to_csv(f, header=False)
print("Predicting COMPLETE!!")
regression = False,
max_epochs = 2, # Number of epochs the training will be done. Higher the value better trained but also risk of overfitting
verbose=1,
)
# Train the network on the training data - Similar to Scikit-learn fit function
net1.fit(train_x_final,train_y_final)
# In[8]:
# test_check = np.asarray(test, dtype = np.float32)
# test_check = test_check.reshape(-1,1,28,28)
# test_check
predicted = net1.predict_proba(test_check)
predicted_nn = pd.DataFrame(predicted, index = test.index, columns = list('0123456789'))
predicted_nn.head(10)
predicted_nn.to_csv("predicted_nn_iter2_probab.csv")
# In[20]:
from lasagne.objectives import multiclass_hinge_loss
# In[ ]:
# scaler = preprocessing.StandardScaler()
# train_final_new = scaler.fit_transform(train_final.ix[:,1:])
dropout1_p=0.35,
dense2_num_units=180,
dropout2_p=0.1,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=0.0003,
# eval_size=0.0,
# objective_loss_function = binary_accuracy,
verbose=1,
max_epochs=1,
)
for i in range(epochs):
net1.fit(X[:split], y[:split])
pred = net1.predict_proba(X[split:])[:, 1]
val_auc[i] = roc_auc_score(y[split:], pred)
test = pd.read_csv("test.csv")
y2 = test.QuoteNumber.values
test = test.drop(["QuoteNumber"], axis=1)
# Lets take out some dates
test["Date"] = pd.to_datetime(pd.Series(test["Original_Quote_Date"]))
test = test.drop("Original_Quote_Date", axis=1)
test["Year"] = test["Date"].apply(lambda x: int(str(x)[:4]))
test["Month"] = test["Date"].apply(lambda x: int(str(x)[5:7]))
test["weekday"] = test["Date"].dt.dayofweek
test = train.drop("Date", axis=1)
remainder_test_points = len(X_test) % NO_TIME_POINTS
no_rows = total_test_time_points * NO_TIME_POINTS
X_test = X_test[0:no_rows, :]
X_test = X_test.transpose()
X_test_Samples = np.split(X_test, total_test_time_points, axis=1)
X_test = np.asarray(X_test_Samples)
###########################################################################
#######get predictions and write to files for series 9 and series 10#######
print("Testing subject%d...." %(subject))
params = net.get_all_params_values()
learned_weights = net.load_params_from(params)
probabilities = net.predict_proba(X_test)
sub9 = 'subj{0}_series{1}'.format(subject, 9)
data_len9 = test_dict[sub9]
total_time_points9 = data_len9 // NO_TIME_POINTS
remainder_data9 = data_len9 % NO_TIME_POINTS
sub10 = 'subj{0}_series{1}'.format(subject, 10)
data_len10 = test_dict[sub10]
total_time_points10 = data_len10 // NO_TIME_POINTS
remainder_data10 = data_len10 % NO_TIME_POINTS
total_test_points = total_time_points9+total_time_points10
for i, p in enumerate(probabilities):
if i != total_test_points:
dense1_num_units=512,
dropout1_p=0.35,
dense2_num_units=180,
dropout2_p=0.1,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=0.0003,
#eval_size=0.0,
# objective_loss_function = binary_accuracy,
verbose=1,
max_epochs=1)
for i in range(epochs):
net1.fit(X[:split], y[:split])
pred = net1.predict_proba(X[split:])[:, 1]
val_auc[i] = roc_auc_score(y[split:], pred)
test = pd.read_csv('test.csv')
y2 = test.QuoteNumber.values
test = test.drop(['QuoteNumber'], axis=1)
# Lets take out some dates
test['Date'] = pd.to_datetime(pd.Series(test['Original_Quote_Date']))
test = test.drop('Original_Quote_Date', axis=1)
test['Year'] = test['Date'].apply(lambda x: int(str(x)[:4]))
test['Month'] = test['Date'].apply(lambda x: int(str(x)[5:7]))
test['weekday'] = test['Date'].dt.dayofweek
test = train.drop('Date', axis=1)
("input", InputLayer),
("dense0", DenseLayer),
("dropout", DropoutLayer),
("dense1", DenseLayer),
("output", DenseLayer),
]
net0 = NeuralNet(
layers=layers0,
input_shape=(None, num_features),
dense0_num_units=100,
dropout_p=0.5,
dense1_num_units=100,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.05,
update_momentum=0.9,
eval_size=0.2,
verbose=1,
max_epochs=20,
)
net0.fit(train_X, train_y)
y_prob = net0.predict_proba(check_X)
print ("LogLoss {score}".format(score=log_loss(check_y, y_prob)))
for dx in range(-6, 6):
for dy in range(-6, 6):
if nuclei_map[limit(i + dx, 0, SIZE - 1), limit(j + dy, 0, SIZE - 1)]:
cover = True
for xx, yy in coords:
if i == xx and j == yy:
print ("Found coordinate of mitosis: ", (xx, yy))
print ("Cover = ", cover)
if cover:
patches.append((i, j))
if len(patches) >= BUFFER:
print ("Evaluating: ", mitosis_area, nuclei_area)
patches2 = get_patches(patches)
prob = nn.predict_proba(patches2)[:, 1]
nuclei_area += np.sum(prob)
mitosis_area += len(prob)
patches = []
if len(patches) >= 1:
print ("Evaluating!")
patches2 = get_patches(patches)
prob = nn.predict_proba(patches2)[:, 1]
nuclei_area += np.sum(prob)
mitosis_area += len(prob)
patches = []
count = 0
print("Starting model combination...")
for startx in range(4):
for starty in range(4):
for rotate in range(4):
net.batch_iterator_test = AugmentedBatchIterator(batch_size=128,
crop_size=4,
testing=True,
startx=startx,
starty=starty,
rotate=rotate)
y_pred_valid[:, count] = net.predict_proba(X_valid)[:, 1]
count += 1
print("Iteration: {} / 64".format(count))
combine = bmc.BMC()
combine.fit(y_pred_valid, y_valid)
print("Validation set done.")
X_test = np.load("../data/sdss_test_images.npy")
y_test = np.load("../data/sdss_test_labels.npy")
for i in range(5):
X_test[:, i, :, :] = renormalize(X_test[:, i, :, :])
)
nn.fit(X_train,y_train)
X_, X_val, y_, y_val = train_test_split(X_train, y_train, test_size = 25000, random_state = 13)
del X_
del y_
xgb = xgboost.XGBClassifier(max_depth = 15, n_estimators = 200,
objective='multi:softprob', subsample = .80, colsample_bytree=.5)
xgb.fit(X_train, y_train, eval_set = [(X_val, y_val)], eval_metric = 'mlogloss', early_stopping_rounds=25)
y_xgb_train_predictions = xgb.predict_proba(X_train)
y_nn_train_predictions = nn.predict_proba(X_train)
X_ensembl_train = np.concatenate((y_xgb_train_predictions, y_nn_train_predictions), axis = 1)
y_nn_test_predictions = nn.predict_proba(X_test)
y_xgb_test_predictions = xgb.predict_proba(X_test)
col_names = ['TripType_' + str(c) for c in enc.classes_.astype('int')]
submission = pd.DataFrame(np.round(y_nn_test_predictions, 4), index=y_df[pd.isnull(y_df.TripType)].index, columns = col_names)
submission.reset_index(inplace = True)
submission.to_csv('Walmart_ensembl_NN_10000Features-Notebook.csv', index=False)
submission = pd.DataFrame(np.round(y_xgb_test_predictions, 4), index=y_df[pd.isnull(y_df.TripType)].index, columns = col_names)
submission.reset_index(inplace = True)
class TwoLayerNeuralNetwork(BaseEstimator):
def __init__(self, **params):
self.seed = params.get('seed', 777)
# Set random seed
random.seed(self.seed)
np.random.seed(self.seed)
# Prepare network parameters
network_params = {}
# Input shape
network_params['input_shape'] = (None, params['num_features'])
# Prepare layers
network_params['layers'] = []
for param_layer in params['layers']:
network_params['layers'].append((
param_layer['name'],
dynamic_load(param_layer['class'])
))
for k in param_layer.keys():
if k in ['name', 'class']:
continue
name = "{0}_{1}".format(param_layer['name'], k)
network_params[name] = param_layer[k]
# Prepare outputs
network_params['output_nonlinearity'] = (
dynamic_load(params['output_nonlinearity']))
# Prepare updates
if 'update' in params:
network_params['update'] = dynamic_load(params['update'])
if 'update_learning_rate' in params:
update_learning_rate = params['update_learning_rate']
network_params['update_learning_rate'] = (
theano.shared(np.cast['float32'](update_learning_rate))
)
if 'adjust_variable' in params:
if params['adjust_variable'] == 1:
adjuster_start = params.get('adjuster_start', 0.01)
adjuster_stop = params.get('adjuster_stop', 0.001)
network_params['on_epoch_finished'] = [
AdjustVariable('update_learning_rate',
start=adjuster_start,
stop=adjuster_stop)
]
# Other parameters
network_params['eval_size'] = params.get('eval_size', 0.01)
network_params['verbose'] = params.get('verbose', 1)
network_params['max_epochs'] = params.get('max_epochs', 100)
# Create a clf object
self.clf = NeuralNet(**network_params)
def fit(self, X_train, y_train):
self.clf.fit(X_train, y_train)
def predict_proba(self, X_test):
return self.clf.predict_proba(X_test)
