def compare():
logger.log('Compare all features, 6 features and 5 features')
mse_all = hist1['Q1']['a']['val_mse']
best_6 = val_mse[[c for c in df.columns
if len(c.split(",")) == 1]].idxmin()
mse_6 = hist2['Q2'][best_6]['val_mse']
best_5 = val_mse[[c for c in df.columns
if len(c.split(",")) == 2]].idxmin()
mse_5 = hist2['Q2'][best_5]['val_mse']
f, ax = plt.subplots(figsize=(10, 5))
plt.plot(mse_all, label='all features')
plt.plot(mse_6, label='6 features')
plt.plot(mse_5, label='5 features')
plt.xlabel('epoch', fontsize=18)
plt.ylabel('mean squared error', fontsize=18)
plt.legend(loc='upper right', fontsize=18)
plt.title('MSE for Different Numbers of Input Features',
fontsize=20,
pad=20)
plt.xlim(1000, 5000)
plt.ylim(0.0058, 0.010)
plt.xticks(fontsize=12, wrap=True)
plt.yticks(fontsize=12, wrap=True)
plt.tight_layout()
plt.savefig('result/BQ2_compare.png')
logger.log('Saved result to \"result/BQ2_compare.png\"')
final = {
'mse_converge': {
"all_features": loss_converge_epoch(mse_all),
"6_features": loss_converge_epoch(mse_6),
"5_features": loss_converge_epoch(mse_5)
},
'final_mse': {
"all_features": training_result(mse_all, mode='loss'),
"6_features": training_result(mse_6, mode='loss'),
"5_features": training_result(mse_5, mode='loss')
}
}
final_df = pd.DataFrame(final)
final_df.to_csv('result/BQ2_result.csv')
logger.log('Saved result to \"result/BQ2_result.csv\"')
# update hyperparameter
result = min(final['final_mse'].keys(),
key=lambda x: final['final_mse'][x])
if result == 'all_features':
removed = []
elif result == '6_features':
removed = list(map(int, best_6.split(",")))
elif result == '5_features':
removed = list(map(int, best_5.split(",")))
hyperparameters['input_shape'] = (len(columns) - len(removed), )
hyperparameters['removed'] = removed
write_dict_to_json(hyperparameters, args.params)
logger.log('Training...')
X_train, y_train = dataset.get_train()
X_test, y_test = dataset.get_test()
with tqdm(total=epochs) as pbar:
update = tf.keras.callbacks.LambdaCallback(
on_epoch_end=lambda batch, logs: pbar.update(1)
)
hist = model.fit(
x=X_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=0,
callbacks=[update]
)
histories['Q5'] = {
'accuracy': hist.history['accuracy'],
'val_accuracy': hist.history['val_accuracy'],
'loss': hist.history['loss'],
'val_loss': hist.history['val_loss']
}
# output to json
logger.log('Saving result to \"result/AQ5.json\"')
write_dict_to_json(filter_dict(histories, ['seed', 'Q5']), 'result/AQ5.json')
logger.end('Stopped q5_train.py')
logger.log('Training...')
X_train, y_train = dataset.get_train()
X_test, y_test = dataset.get_test()
with tqdm(total=epochs) as pbar:
update = tf.keras.callbacks.LambdaCallback(
on_epoch_end=lambda batch, logs: pbar.update(1)
)
hist = model.fit(
x=X_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=0,
callbacks=[update]
)
histories['Q5'] = {
'accuracy': hist.history['accuracy'],
'val_accuracy': hist.history['val_accuracy'],
'loss': hist.history['loss'],
'val_loss': hist.history['val_loss']
}
# output to json
logger.log('Saving result to \"result/additional.json\"')
write_dict_to_json(filter_dict(histories, ['seed', 'Q5']), 'result/additional.json')
logger.end('Stopped additional_train.py')
num_classes = 3
epochs = 1000
batch_size = 32
num_neurons = 10
alpha = 0.01
beta = 1e-6
hyperparameters = {
"input_shape": input_shape,
"num_classes": num_classes,
"batch_size": batch_size,
"num_neurons": num_neurons,
"alpha": alpha,
"beta": beta
}
write_dict_to_json(hyperparameters, args.params)
# create and compile model
logger.log('Creating model...')
model = tf.keras.Sequential([
tf.keras.layers.InputLayer(input_shape=input_shape),
tf.keras.layers.Dense(num_neurons,
activation='relu',
kernel_regularizer=tf.keras.regularizers.l2(l=beta)),
tf.keras.layers.Dense(num_classes,
activation='softmax',
kernel_regularizer=tf.keras.regularizers.l2(l=beta))
])
model.compile(
optimizer=tf.keras.optimizers.SGD(learning_rate=alpha),
def cross_validation():
input_shape = hyperparameters['input_shape']
num_classes = hyperparameters['num_classes']
epochs = 500
batch_sizes = [4, 8, 16, 32, 64]
num_neurons = hyperparameters['num_neurons']
alpha = hyperparameters['alpha']
histories['Q2'] = {
'cv': {
'accuracy': {batch: [] for batch in batch_sizes},
'time': {batch: [] for batch in batch_sizes},
},
'optimal': dict()
}
logger.log('Starting cross validation...')
X, y = dataset.get_train()
X_test, y_test = dataset.get_test()
for fold, (train_index, valid_index) in enumerate(dataset.get_kfold(), start=1):
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
for batch in batch_sizes:
logger.log('Fold %s Batch Size %s' % (fold, batch))
with tqdm(total=epochs, desc='Fold %s Batch Size %s' % (fold, batch)) as pbar:
update = tf.keras.callbacks.LambdaCallback(
on_epoch_end=lambda batch, logs: pbar.update(1)
)
model = tf.keras.Sequential([
tf.keras.layers.InputLayer(input_shape=input_shape),
tf.keras.layers.Dense(num_neurons, activation='relu'),
tf.keras.layers.Dense(num_classes, activation='softmax')
])
model.compile(
optimizer=tf.keras.optimizers.SGD(learning_rate=alpha),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['accuracy']
)
start = time.time()
history = model.fit(
x=X_train,
y=y_train,
batch_size=batch,
epochs=epochs,
validation_data=(X_valid, y_valid),
verbose=0,
callbacks=[update]
)
end = time.time()
histories['Q2']['cv']['accuracy'][batch].append(history.history['val_accuracy'])
histories['Q2']['cv']['time'][batch].append((end - start) / epochs)
batch_epoch = pd.DataFrame(histories['Q2']['cv']['accuracy'])
time_per_epoch = pd.DataFrame(histories['Q2']['cv']['time'])
epoch_to_converge = batch_epoch.applymap(lambda x: acc_converge_epoch(x))
cv_accuracy = batch_epoch.applymap(lambda x: training_result(x)).mean()
total_time_to_converge = (epoch_to_converge * time_per_epoch).mean()
def standardize(arr):
return (arr - arr.mean()) / arr.std()
def deciding_factor(total_time, acc):
time_score = np.exp(-standardize(total_time))
acc_score = np.exp(standardize(acc))
print(acc_score * time_score)
return int((acc_score * time_score).idxmax())
hyperparameters['batch_size'] = deciding_factor(total_time_to_converge, cv_accuracy)
logger.log('Optimal batch size: %s' % hyperparameters['batch_size'])
write_dict_to_json(hyperparameters, args.params)
logger.log('Done cross validation')
fold=5)
X_train, y_train = dataset.get_train()
X_test, y_test = dataset.get_test()
# Defining hyperparameters
try:
hyperparameters = read_json_to_dict(args.params)
except:
hyperparameters = {
"input_shape": (7, ),
"batch_size": 8,
"num_neurons": 10,
"alpha": 1e-3,
"beta": 1e-3
}
write_dict_to_json(hyperparameters, args.params)
epochs = 5000
batch_size = hyperparameters['batch_size']
num_neurons = hyperparameters['num_neurons']
alpha = hyperparameters['alpha']
beta = hyperparameters['beta']
# Histories of results
histories = {'seed': SEED, 'columns': df.columns[1:8].tolist(), 'Q2': dict()}
write_dict_to_json(filter_dict(histories, ['seed', 'columns', 'Q2']),
'result/BQ2.json')
# RFE functions
def remove_features(f_arr, remove):
def cross_validation():
input_shape = hyperparameters['input_shape']
num_classes = hyperparameters['num_classes']
epochs = 500
batch_size = hyperparameters['batch_size']
num_neurons = hyperparameters['num_neurons']
alpha = hyperparameters['alpha']
beta = [0, 1e-3, 1e-6, 1e-9, 1e-12]
histories['Q4'] = {
'cv': {
'accuracy': {b: [] for b in beta},
'time': {b: [] for b in beta},
},
'optimal': dict()
}
logger.log('Starting cross validation...')
X, y = dataset.get_train()
X_test, y_test = dataset.get_test()
for fold, (train_index, valid_index) in enumerate(dataset.get_kfold(), start=1):
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
for b in beta:
logger.log('Fold %s Decay %s' % (fold, b))
with tqdm(total=epochs, desc='Fold %s Decay %s' % (fold, b)) as pbar:
update = tf.keras.callbacks.LambdaCallback(
on_epoch_end=lambda batch, logs: pbar.update(1)
)
model = tf.keras.Sequential([
tf.keras.layers.InputLayer(input_shape=input_shape),
tf.keras.layers.Dense(num_neurons, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=b)),
tf.keras.layers.Dense(num_classes, activation='softmax', kernel_regularizer=tf.keras.regularizers.l2(l=b))
])
model.compile(
optimizer=tf.keras.optimizers.SGD(learning_rate=alpha),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['accuracy']
)
start = time.time()
history = model.fit(
x=X_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_valid, y_valid),
verbose=0,
callbacks=[update]
)
end = time.time()
histories['Q4']['cv']['accuracy'][b].append(history.history['val_accuracy'])
histories['Q4']['cv']['time'][b].append((end - start) / epochs)
beta_epoch = pd.DataFrame(histories['Q4']['cv']['accuracy'])
cv_accuracy = beta_epoch.applymap(lambda x: training_result(x)).mean()
hyperparameters['beta'] = float(cv_accuracy.idxmax())
write_dict_to_json(hyperparameters, args.params)
logger.log('Done cross validation')
logger.log('Loading dataset from \"' + args.data + '\"...')
df = pd.read_csv(args.data)
# Defining hyperparameters
try:
hyperparameters = read_json_to_dict(args.params)
except:
hyperparameters = {
"input_shape": (7, ),
"features_left": df.columns[1:8].tolist(),
"batch_size": 8,
"num_neurons": 10,
"alpha": 1e-3,
"beta": 1e-3
}
write_dict_to_json(hyperparameters, args.params)
epochs = 10000
num_neurons = 50
dropout = 0.2
batch_size = hyperparameters['batch_size']
alpha = hyperparameters['alpha']
beta = hyperparameters['beta']
# Histories of results
histories = {'seed': SEED, 'Q3': dict()}
features_left = hyperparameters['features_left']
input_shape = hyperparameters['input_shape']
dataset = PreprocessDataset(df=df,
model.compile(
optimizer=tf.keras.optimizers.Adam(lr=0.001),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=[
tf.keras.metrics.SparseTopKCategoricalAccuracy(k=1,
name='top1_accuracy',
dtype=None),
tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5,
name='top5_accuracy',
dtype=None)
])
model.trainable = False
results = model.evaluate(test_ds,
batch_size=batch_size,
verbose=0,
return_dict=True)
print(results)
# Save results
if not os.path.exists('./log'):
os.mkdir('./log')
if not os.path.exists(f'./log/{model.name}'):
os.mkdir(f'./log/{model.name}')
log_dir = f'./log/{model.name}/' + args.model_weights.split(
'/')[-1] + '_test.log'
write_dict_to_json(results, log_dir)
logger.log('Saved result to \"' + log_dir + '\"')
logger.end('Done model_test.py')
