def __init__(self, dataset_name, model_name, optimizer_name, trial_num):
"""
:param dataset_name: name of the dataset
:type dataset_name: str
:param model_name: name of the model
:type model_name: str
:param optimizer_name: name of the optimizer
:type optimizer_name: str
:param trial_num: current number of repeated trials
:type trial_num: int
"""
# get optimized hyperparameters
with open(
f'../params/{dataset_name}_{model_name}_{optimizer_name}/result.json'
) as f:
params = json.load(f)
# get instances
self.dataset = Datasets.get(dataset_name)
self.model = Models.get(model_name, dataset=self.dataset)
self.optimizer = Optimizers.get(optimizer_name, params=params)
# get config
with open('./config.json') as f:
config = json.load(f)
# get constants
c = config['constants'][dataset_name][model_name]
self.loss = c['loss']
self.batch_size = c['batch_size']
self.epochs = c['epochs']
# configure and initialize directory
d = self.main_dir = f'../data/{dataset_name}_{model_name}_{optimizer_name}/trial{trial_num}'
if os.path.exists(d):
shutil.rmtree(d)
os.makedirs(d)
# configure hyperdash experiment
self.hd_exp = HyperdashExperiment(
f'{dataset_name}',
api_key_getter=lambda: config['hyperdash']['api_key'])
self.hd_exp.param('dataset_name', dataset_name)
self.hd_exp.param('model_name', model_name)
self.hd_exp.param('optimizer_name', optimizer_name)
self.hd_exp.param('trial_num', trial_num)
for k, v in params.items():
self.hd_exp.param(k, v)
# set callbacks
self.callbacks = [
Hyperdash(['accuracy', 'loss', 'val_accuracy', 'val_loss'],
self.hd_exp),
TensorBoard(log_dir=f'{self.main_dir}/tensorboard'),
TimeLogger(filename=f'{self.main_dir}/time.csv'),
CSVLogger(filename=f'{self.main_dir}/result.csv', append=True)
]
def objective(self, params):
"""
objective function to optimize
:param params: hyperparamters for optimizer
:return: maximum validation accuracy
:rtype: float
"""
# get instances
dataset = Datasets.get(self.dataset_name)
model = Models.get(self.model_name, dataset=dataset)
optimizer = Optimizers.get(self.optimizer_name, params=params)
# configure hyperdash experiment
hd_exp = HyperdashExperiment(
f'{self.dataset_name}',
api_key_getter=lambda: self.config['hyperdash']['api_key'])
hd_exp.param('dataset_name', self.dataset_name)
hd_exp.param('model_name', self.model_name)
hd_exp.param('optimizer_name', self.optimizer_name)
for k, v in params.items():
hd_exp.param(k, v)
# set callbacks
callbacks = [
Hyperdash(['accuracy', 'loss', 'val_accuracy', 'val_loss'],
hd_exp),
EarlyStopping('val_accuracy',
patience=10,
min_delta=0.01,
verbose=1),
TerminateOnNaN()
]
# get data
(x_train, y_train), *_ = dataset.get_batch()
# start learning
model.compile(loss=self.loss,
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs,
callbacks=callbacks,
validation_split=0.2,
verbose=2)
# stop hyperdash experiment
hd_exp.end()
# return maximum validation accuracy
val_accuracy = np.array(history.history['val_accuracy'])
return max(val_accuracy) * (-1)
def main(args):
"""Execute a task based on the given command-line arguments.
This function is the main entry-point of the program. It allows the
user to extract features, train a model, compute predictions, and
evaluate predictions using the command-line interface.
"""
from datasets import Datasets
datasets = Datasets(args.dataset_path)
if args.command == 'extract':
extract(datasets.get(args.dataset), args)
elif args.command == 'train':
train(datasets.get('training'), args)
elif args.command == 'predict':
predict(datasets.get(args.dataset), args)
elif args.command == 'evaluate':
if isinstance(args.training_id, list):
evaluate_all(datasets.get('test'), args)
else:
evaluate(datasets.get('test'), args)