def main():
(X_train, y_train), (X_test,
y_test) = p1b2.load_data(n_cols=FEATURE_SUBSAMPLE)
input_dim = X_train.shape[1]
output_dim = y_train.shape[1]
model = Sequential()
model.add(
Dense(LAYERS[0],
input_dim=input_dim,
activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
for layer in LAYERS[1:]:
if layer:
if DROP:
model.add(Dropout(DROP))
model.add(
Dense(layer,
activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
model.add(Dense(output_dim, activation=ACTIVATION))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print(model.summary())
ext = extension_from_parameters()
checkpointer = ModelCheckpoint(filepath='model' + ext + '.h5',
save_best_only=True)
history = BestLossHistory()
model.fit(X_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
validation_split=0.2,
callbacks=[history, checkpointer])
y_pred = history.best_model.predict(X_test)
print('best_val_loss={:.5f} best_val_acc={:.5f}'.format(
history.best_val_loss, history.best_val_acc))
print('Best model saved to: {}'.format('model' + ext + '.h5'))
scores = p1b2.evaluate(y_pred, y_test)
print('Evaluation on test data:', scores)
submission = {
'scores': scores,
'model': model.summary(),
'submitter': 'Developer Name'
}
def main():
(X_train, y_train), (X_test, y_test) = p1b2.load_data(n_cols=FEATURE_SUBSAMPLE)
input_dim = X_train.shape[1]
output_dim = y_train.shape[1]
model = Sequential()
model.add(Dense(LAYERS[0], input_dim=input_dim,
activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
for layer in LAYERS[1:]:
if layer:
if DROP:
model.add(Dropout(DROP))
model.add(Dense(layer, activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
model.add(Dense(output_dim, activation=ACTIVATION))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
print(model.summary())
ext = extension_from_parameters()
checkpointer = ModelCheckpoint(filepath='model'+ext+'.h5', save_best_only=True)
history = BestLossHistory()
model.fit(X_train, y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
validation_split=0.2,
callbacks=[history, checkpointer])
y_pred = history.best_model.predict(X_test)
print('best_val_loss={:.5f} best_val_acc={:.5f}'.format(history.best_val_loss, history.best_val_acc))
print('Best model saved to: {}'.format('model'+ext+'.h5'))
scores = p1b2.evaluate(y_pred, y_test)
print('Evaluation on test data:', scores)
submission = {'scores': scores,
'model': model.summary(),
'submitter': 'Developer Name' }
def recordSoftmaxProbabilities(X_train=None,
y_train=None,
X_test=None,
y_test=None,
DeterministicResults=False,
fileName=None):
if (DeterministicResults):
__setSession()
wb = Workbook()
# =====create sheet1 and add headers====
sheetToRecordTrainValidTestLossAndAccuracy = wb.add_sheet('Sheet 1')
sheetToRecordTrainValidTestLossAndAccuracy.write(0, 0, 'ValidationLoss')
sheetToRecordTrainValidTestLossAndAccuracy.write(0, 1, 'TestLoss')
sheetToRecordTrainValidTestLossAndAccuracy.write(0, 2, 'Accuracy')
for x in range(1, 101):
if X_train is None:
(X_train,
y_train), (X_test,
y_test) = p1b2.load_data(n_cols=FEATURE_SUBSAMPLE)
input_dim = X_train.shape[1]
output_dim = y_train.shape[1]
model = Sequential()
model.add(
Dense(LAYERS[0],
input_dim=input_dim,
activation="sigmoid",
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
for layer in LAYERS[1:]:
if layer:
if DROP:
model.add(Dropout(DROP))
model.add(
Dense(layer,
activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
model.add(Dense(output_dim, activation='softmax'))
#Next the model would be compiled. Compiling the model takes two parameters: optimizer and loss
#https: // towardsdatascience.com / building - a - deep - learning - model - using - keras - 1548ca149d37
#https://towardsdatascience.com/sequence-models-by-andrew-ng-11-lessons-learned-c62fb1d3485b
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print("Model Summary:", model.summary())
ext = extension_from_parameters()
checkpointer = ModelCheckpoint(filepath='model' + ext + '.h5',
save_best_only=True)
history = BestLossHistory()
trainingResults = model.fit(X_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
validation_split=0.2,
callbacks=[history, checkpointer])
y_pred = history.best_model.predict(X_test)
predictedOutputs = model.predict_classes(X_test)
scores = p1b2.evaluate(y_pred, y_test)
#Confusion Matrix
#cnf_matrix = confusion_matrix(y_test_SingleColumn, predictedOutputs)
#print("Confusion Matrix = ", cnf_matrix)
#ROC curve
# keep probabilities for the positive outcome only
#ns_probs = [0 for _ in range(len(y_test_SingleColumn))]
#lr_probs = y_pred[:, 0]
#print("Faqeer = ", lr_probs)
# calculate scores
#ns_auc = roc_auc_score(y_test_SingleColumn, ns_probs)
#lr_auc = roc_auc_score(y_test_SingleColumn, lr_probs)
#print('No Skill: ROC AUC=%.3f' % (ns_auc))
#print('Logistic: ROC AUC=%.3f' % (lr_auc))
#Print Other Results
testResults = model.evaluate(X_test, y_test, batch_size=BATCH_SIZE)
#print('Evaluation on test data:', scores)
#print('Test Scores [Test Loss, Test Accuracy] = ', testResults[0])
#print('Loss: ', np.amin(trainingResults.history['loss']),'Accuracy: ',np.amin(trainingResults.history['accuracy']),'Val_Loss: ',np.amin(trainingResults.history['val_loss']),'Val_Accuracy :',np.amin(trainingResults.history['val_accuracy']))
#print('best_val_loss={:.5f} best_val_acc={:.5f}'.format(history.best_val_loss, history.best_val_acc))
#print('Best model saved to: {}'.format('model'+ext+'.h5'))
# ======Save Training loss,Validation(Best model) loss, test loss and Accuracy
#sheetToRecordTrainValidTestLossAndAccuracy.write(x, 0, str(round(np.amin(trainingResults.history['loss']), 3)))
sheetToRecordTrainValidTestLossAndAccuracy.write(
x, 0, str(round(history.best_val_loss, 3)))
sheetToRecordTrainValidTestLossAndAccuracy.write(
x, 1, str(round(testResults[0], 3)))
sheetToRecordTrainValidTestLossAndAccuracy.write(x, 2, str(scores))
# ===========================================================================
# =====Save Instance level outputs against each experiment/iteration over for Each Class=====
# =====create sheet2 and add headers====
sheetToRecordInstanceLevelOutput = wb.add_sheet('IterationNo' + str(x))
sheetToRecordInstanceLevelOutput.write(1, 0, 'InputFeatures')
sheetToRecordInstanceLevelOutput.write(1, 1,
'Expected_OR_ActualOutput')
sheetToRecordInstanceLevelOutput.write(1, 2, 'PredictedOutput')
sheetToRecordInstanceLevelOutput.write(1, 3, 'Probabilities')
sheetToRecordInstanceLevelOutput.write(1, 4, 'MaxProbability')
startRowToBeInserted = 2
for x in range(X_test.shape[0]):
# print("ddd = ", X_test[x])
sheetToRecordInstanceLevelOutput.write(
startRowToBeInserted, 0,
'Test Data Input Features') # str(X_test[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 1,
str(y_test[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 2,
str(predictedOutputs[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 3,
str(y_pred[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 4,
str(np.amax(y_pred[x])))
startRowToBeInserted = startRowToBeInserted + 1
# ==============================================================================
submission = {
'scores': scores,
'model': model.summary(),
'submitter': 'Developer Name'
}
if fileName != None:
wb.save(fileName) # .xls
else:
wb.save("Default.xls") # .xls
# print('Submitting to leaderboard...')
# leaderboard.submit(submission)
__resetSeed()
# return history.best_model
return scores
def test():
(X_train, y_train), (X_test,
y_test) = p1b2.load_data(n_cols=FEATURE_SUBSAMPLE)
#dist = pd.DataFrame(X_train)
print(y_test.shape)
def mainFeatureSelection(X_train=None,
y_train=None,
X_test=None,
y_test=None,
DeterministicResults=False):
if (DeterministicResults):
__setSession()
if X_train is None:
(X_train, y_train), (X_test,
y_test) = p1b2.load_data(n_cols=FEATURE_SUBSAMPLE)
input_dim = X_train.shape[1]
output_dim = y_train.shape[1]
print("X Train: ", X_train)
print("Y Train: ", y_train)
model = Sequential()
model.add(
Dense(LAYERS[0],
input_dim=input_dim,
activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
for layer in LAYERS[1:]:
if layer:
if DROP:
model.add(Dropout(DROP))
model.add(
Dense(layer,
activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
model.add(Dense(output_dim, activation=ACTIVATION))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print(model.summary())
ext = extension_from_parameters()
checkpointer = ModelCheckpoint(filepath='model' + ext + '.h5',
save_best_only=True)
history = BestLossHistory()
model.fit(X_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
validation_split=0.2,
callbacks=[history, checkpointer])
y_pred = history.best_model.predict(X_test)
predictedOutputs = model.predict_classes(X_test)
#print("TestDataX = ", X_test)
#print("TestDataY = ", y_test)
#i=0
#j=1
#for x in np.nditer(X_test, flags = ['external_loop'], order = 'C'):
# print("Instance = ", X_test[i:j], " --> Prediciton = ", history.best_model.predict(np.array(X_test[i:j])))
# i = i + 1
# j = j + 1
#print("Loop Iterations : ", x)
#print("Y_Pred = " , y_pred)
#print("PredictedOutputs = ", predictedOutputs)
scores = p1b2.evaluate(y_pred, y_test)
print('Evaluation on test data:', scores)
print('best_val_loss={:.5f} best_val_acc={:.5f}'.format(
history.best_val_loss, history.best_val_acc))
print('Best model saved to: {}'.format('model' + ext + '.h5'))
submission = {
'scores': scores,
'model': model.summary(),
'submitter': 'Developer Name'
}
# print('Submitting to leaderboard...')
# leaderboard.submit(submission)
__resetSeed()
#return history.best_model
return scores
from __future__ import print_function import numpy as np from sklearn.model_selection import cross_val_score from xgboost import XGBClassifier import p1b2 # (X_train, y_train), (X_test, y_test) = p1b2.load_data(n_cols=100) (X_train, y_train), (X_test, y_test) = p1b2.load_data() y_train = y_train.argmax(axis=1) y_test = y_test.argmax(axis=1) clf = XGBClassifier(max_depth=3, n_estimators=100, learning_rate=0.05) scores = cross_val_score(clf, X_train, y_train, cv=5) print(scores) print(np.mean(scores))
def setUpClass(self):
self._srcModel = p1b2_baseline_keras2.main(DeterministicResults=True)
(self.X_train, self.y_train), (self.X_test,
self.y_test) = p1b2.load_data()
self._origPredictions = self._srcModel.predict_classes(self.X_test)
def main():
# Get command-line parameters
parser = get_p1b2_parser()
args = parser.parse_args()
#print('Args:', args)
# Get parameters from configuration file
fileParameters = p1b2.read_config_file(args.config_file)
#print ('Params:', fileParameters)
# Correct for arguments set by default by neon parser
# (i.e. instead of taking the neon parser default value fall back to the config file,
# if effectively the command-line was used, then use the command-line value)
# This applies to conflictive parameters: batch_size, epochs and rng_seed
if not any("--batch_size" in ag or "-z" in ag for ag in sys.argv):
args.batch_size = fileParameters['batch_size']
if not any("--epochs" in ag or "-e" in ag for ag in sys.argv):
args.epochs = fileParameters['epochs']
if not any("--rng_seed" in ag or "-r" in ag for ag in sys.argv):
args.rng_seed = fileParameters['rng_seed']
# Consolidate parameter set. Command-line parameters overwrite file configuration
gParameters = p1_common.args_overwrite_config(args, fileParameters)
print('Params:', gParameters)
# Determine verbosity level
loggingLevel = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=loggingLevel, format='')
# Construct extension to save model
ext = p1b2.extension_from_parameters(gParameters, '.neon')
# Get default parameters for initialization and optimizer functions
kerasDefaults = p1_common.keras_default_config()
seed = gParameters['rng_seed']
# Load dataset
#(X_train, y_train), (X_test, y_test) = p1b2.load_data(gParameters, seed)
(X_train, y_train), (X_val,
y_val), (X_test,
y_test) = p1b2.load_data(gParameters, seed)
print("Shape X_train: ", X_train.shape)
print("Shape X_val: ", X_val.shape)
print("Shape X_test: ", X_test.shape)
print("Shape y_train: ", y_train.shape)
print("Shape y_val: ", y_val.shape)
print("Shape y_test: ", y_test.shape)
print("Range X_train --> Min: ", np.min(X_train), ", max: ",
np.max(X_train))
print("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val))
print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test))
print("Range y_train --> Min: ", np.min(y_train), ", max: ",
np.max(y_train))
print("Range y_val --> Min: ", np.min(y_val), ", max: ", np.max(y_val))
print("Range y_test --> Min: ", np.min(y_test), ", max: ", np.max(y_test))
input_dim = X_train.shape[1]
num_classes = int(np.max(y_train)) + 1
output_dim = num_classes # The backend will represent the classes using one-hot representation (but requires an integer class as input !)
# Re-generate the backend after consolidating parsing and file config
gen_backend(backend=args.backend,
rng_seed=seed,
device_id=args.device_id,
batch_size=gParameters['batch_size'],
datatype=gParameters['data_type'],
max_devices=args.max_devices,
compat_mode=args.compat_mode)
train = ArrayIterator(X=X_train, y=y_train, nclass=num_classes)
val = ArrayIterator(X=X_val, y=y_val, nclass=num_classes)
test = ArrayIterator(X=X_test, y=y_test, nclass=num_classes)
# Initialize weights and learning rule
initializer_weights = p1_common_neon.build_initializer(
gParameters['initialization'], kerasDefaults, seed)
initializer_bias = p1_common_neon.build_initializer(
'constant', kerasDefaults, 0.)
activation = p1_common_neon.get_function(gParameters['activation'])()
# Define MLP architecture
layers = []
reshape = None
for layer in gParameters['dense']:
if layer:
layers.append(
Affine(nout=layer,
init=initializer_weights,
bias=initializer_bias,
activation=activation))
if gParameters['dropout']:
layers.append(Dropout(keep=(1 - gParameters['dropout'])))
layers.append(
Affine(nout=output_dim,
init=initializer_weights,
bias=initializer_bias,
activation=activation))
# Build MLP model
mlp = Model(layers=layers)
# Define cost and optimizer
cost = GeneralizedCost(p1_common_neon.get_function(gParameters['loss'])())
optimizer = p1_common_neon.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
callbacks = Callbacks(mlp, eval_set=val, metric=Accuracy(), eval_freq=1)
# Seed random generator for training
np.random.seed(seed)
mlp.fit(train,
optimizer=optimizer,
num_epochs=gParameters['epochs'],
cost=cost,
callbacks=callbacks)
# model save
#save_fname = "model_mlp_W_" + ext
#mlp.save_params(save_fname)
# Evalute model on test set
print('Model evaluation by neon: ', mlp.eval(test, metric=Accuracy()))
y_pred = mlp.get_outputs(test)
#print ("Shape y_pred: ", y_pred.shape)
scores = p1b2.evaluate_accuracy(p1_common.convert_to_class(y_pred), y_test)
print('Evaluation on test data:', scores)
def setUpClass(self):
(self.X_train, self.y_train), (self.X_test,
self.y_test) = p1b2.load_data()
def recordSoftmaxProbabilities(X_train = None, y_train = None, X_test = None, y_test = None, DeterministicResults = False,fileName=None,classLabel=''):
if(DeterministicResults):
__setSession()
# Workbook is created
wb = Workbook()
# =====create sheet1 and add headers====
sheetToRecordTrainValidTestLossAndAccuracy = wb.add_sheet('Sheet 1')
sheetToRecordTrainValidTestLossAndAccuracy.write(0, 0, 'Class Label = ' + classLabel)
sheetToRecordTrainValidTestLossAndAccuracy.write(0, 1, 'ValidationLoss')
sheetToRecordTrainValidTestLossAndAccuracy.write(0, 2, 'TestLoss')
sheetToRecordTrainValidTestLossAndAccuracy.write(0, 3, 'Accuracy')
for x in range(1,101):
print('mainToRecordTrainValidateTestLosses:Run===>',x)
if X_train is None:
(X_train, y_train), (X_test, y_test) = p1b2.load_data(False, False)
k = 1; #np.random.randint(1, 100)
b = 10; #np.random.randint(100)
for i in range(X_train.shape[1]):
X_train[:, i] = X_train[:, i] * k + b
X_test[:, i] = X_test[:, i] * k + b
#(X_train, y_train), (X_test, y_test) = p1b2.load_dataWithImportantFeatures(False, False)
#(X_train, y_train), (X_test, y_test) = p1b2.load_dataWithImportantFeatures(n_cols=FEATURE_SUBSAMPLE)
input_dim = X_train.shape[1]
output_dim = y_train.shape[1]
model = Sequential()
model.add(Dense(LAYERS[0], input_dim=input_dim,
activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
for layer in LAYERS[1:]:
if layer:
if DROP:
model.add(Dropout(DROP))
model.add(Dense(layer, activation=ACTIVATION,
kernel_regularizer=l2(PENALTY),
activity_regularizer=l2(PENALTY)))
#model.add(Dense(output_dim, activation=ACTIVATION))
model.add(Dense(output_dim, activation=outputLayerActivation)) #Added by Faqeer
#Next the model would be compiled. Compiling the model takes two parameters: optimizer and loss
#https: // towardsdatascience.com / building - a - deep - learning - model - using - keras - 1548ca149d37
#https://towardsdatascience.com/sequence-models-by-andrew-ng-11-lessons-learned-c62fb1d3485b
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
print("Model Summary:", model.summary())
ext = extension_from_parameters()
checkpointer = ModelCheckpoint(filepath='model'+ext+'.h5', save_best_only=True)
history = BestLossHistory()
trainingResults = model.fit(X_train, y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
validation_split=0.2,
callbacks=[history, checkpointer])
y_pred = history.best_model.predict(X_test)
predictedOutputs = model.predict_classes(X_test)
#print("Y_Pred = " , y_pred)
#print("PredictedOutputs = ", predictedOutputs)
scores = p1b2.evaluate(y_pred, y_test)
testResults = model.evaluate(X_test,y_test,batch_size=BATCH_SIZE)
print('Evaluation on test data:', scores)
print('Loss: ', np.amin(trainingResults.history['loss']),'Accuracy: ',np.amin(trainingResults.history['accuracy']),'Val_Loss: ',np.amin(trainingResults.history['val_loss']),'Val_Accuracy :',np.amin(trainingResults.history['val_accuracy']))
print('best_val_loss={:.5f} best_val_acc={:.5f}'.format(history.best_val_loss, history.best_val_acc))
print('Test Scores [Test Loss] = ', testResults[0])
print('Evaluation on test data:', scores)
#print('Best model saved to: {}'.format('model'+ext+'.h5'))
#======Save Training loss,Validation(Best model) loss, test loss and Accuracy
sheetToRecordTrainValidTestLossAndAccuracy.write(x, 0, str(round(np.amin(trainingResults.history['loss']),3)))
sheetToRecordTrainValidTestLossAndAccuracy.write(x, 1, str(round(history.best_val_loss,3)))
sheetToRecordTrainValidTestLossAndAccuracy.write(x, 2, str(round(testResults[0],3)))
sheetToRecordTrainValidTestLossAndAccuracy.write(x, 3, str(scores))
#===========================================================================
#=====Save Instance level outputs against each experiment/iteration over for Each Class=====
# =====create sheet2 and add headers====
sheetToRecordInstanceLevelOutput = wb.add_sheet('IterationNo'+str(x)+'_ClassLabel='+classLabel)
sheetToRecordInstanceLevelOutput.write(0, 0, 'Test Data Outputs for Class Label = ' + classLabel)
sheetToRecordInstanceLevelOutput.write(1, 0, 'InputFeatures')
sheetToRecordInstanceLevelOutput.write(1, 1, 'Expected_OR_ActualOutput')
sheetToRecordInstanceLevelOutput.write(1, 2, 'PredictedOutput')
sheetToRecordInstanceLevelOutput.write(1, 3, 'Probabilities')
sheetToRecordInstanceLevelOutput.write(1, 4, 'MaxProbability')
startRowToBeInserted = 2
for x in range(X_test.shape[0]):
# print("ddd = ", X_test[x])
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 0, 'Test Data Input Features')# str(X_test[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 1, str(y_test[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 2, str(predictedOutputs[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 3, str(y_pred[x]))
sheetToRecordInstanceLevelOutput.write(startRowToBeInserted, 4, str(np.amax(y_pred[x])))
startRowToBeInserted = startRowToBeInserted + 1
#==============================================================================
submission = {'scores': scores,
'model': model.summary(),
'submitter': 'Developer Name' }
if fileName!=None:
wb.save(fileName) #.xls
else:
wb.save("Default.xls") # .xls
# print('Submitting to leaderboard...')
# leaderboard.submit(submission)
__resetSeed()
#return history.best_model
return None# history.best_model#scores
