def cv_loop(X, y, model, N):
mean_auc = 0.
for i in range(N):
X_train, X_cv, y_train, y_cv = cross_validation.train_test_split(
X, y, test_size=.20,
random_state = i*SEED)
model.fit(X_train, y_train)
preds = model.predict_proba(X_cv)[:,1]
auc = metrics.auc_score(y_cv, preds)
print "AUC (fold %d/%d): %f" % (i + 1, N, auc)
mean_auc += auc
return mean_auc/N
submission = True
####################
# MEL SPECTRUM #
####################
trainset,validset,testset = Whales.whaledata.get_dataset('melspectrum', tot=submission)
# build and train classifiers for submodels
model = get_conv2D([67,40,1])
get_trainer(model, trainset, validset, 80).main_loop()
# validate model
if not submission:
output = get_output(model,validset)
# calculate AUC using sklearn
AUC = auc_score(validset.get_targets()[:,0],output[:,0])
print AUC
else:
outtestset = get_output(model,testset,-1)
# save test output as submission
np.savetxt(DATA_DIR+'model_conv2net.csv', outtestset[:,0], delimiter=",")
# construct data sets with model output
outtrainset = get_output(model,trainset,-2)
outtestset = get_output(model,testset,-2)
np.save(DATA_DIR+'conv2altout_train', outtrainset)
np.save(DATA_DIR+'conv2altout_test', outtestset)
#########################
