def main(args):
if tf.test.is_gpu_available():
print(bc.OKGREEN + bc.BOLD + '#' * 9 + ' USING GPU ' + '#' * 9 +
bc.ENDC)
else:
print(bc.FAIL + bc.BOLD + '#' * 9 + ' NOT USING GPU ' + '#' * 9 +
bc.ENDC)
# Get agent name
tokens = args.p.split('/')
if args.p[-1] == '/':
assert (tokens.pop() == '')
dir_agent = '-'.join(tokens[-1].split('_')[:-3]) + '.save'
print(dir_agent)
# run this first to avoid failing after huge overhead
model_ok, initial_epoch = model_exists(args.m, dir_agent)
PATH_DIR_SAVE = os.path.join(args.m, dir_agent)
PATH_DIR_CKPT = os.path.join(PATH_DIR_SAVE, 'ckpts')
n_epoch = args.epochs
hypers = {
'lr': 0.00015,
'batch_size': 128,
'hl_activations': [ReLU, ReLU, ReLU, ReLU, ReLU, ReLU],
'hl_sizes': [1024, 1024, 512, 512, 512, 256],
'decay': 0.,
'bNorm': True,
'dropout': True,
'regularizer': None
}
# checking input data format.
if args.p.split('.')[-1] in ['hdf5', 'HDF5']:
f = h5py_cache.File(p,
'r',
chunk_cache_mem_size=1 * 1024**3,
swmr=True)
gen_tr = Gen4h5(f['X_tr'], f['Y_tr'], hypers['batch_size'], False)
gen_va = Gen4h5(f['X_va'], f['Y_va'], hypers['batch_size'], False)
else:
X, Y, mask = CV(args.p)
gen_tr = DataGenerator(X[mask], Y[mask], hypers['batch_size'])
gen_va = DataGenerator(X[~mask], Y[~mask], 1000)
os.makedirs(PATH_DIR_CKPT, exist_ok=True)
# Callbacks: save best & latest models.
callbacks = [
ModelCheckpoint(os.path.join(PATH_DIR_SAVE, 'best.h5'),
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1),
ModelCheckpoint(os.path.join(PATH_DIR_CKPT,
'{epoch:02d}-{val_accuracy:.2f}.h5'),
monitor='val_loss',
verbose=1,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=1),
CSVLogger(os.path.join(PATH_DIR_SAVE, 'training.log'), append=True)
]
m = Mlp(io_sizes=(glb.SIZE_OBS_VEC, glb.SIZE_ACT_VEC),
out_activation=Softmax,
loss='categorical_crossentropy',
metrics=['accuracy'],
**hypers,
verbose=1)
if model_ok:
# continue from previously saved
msg = "Saved model found. Resuming training."
print(bc.OKGREEN + bc.BOLD + msg + bc.ENDC)
h5s = os.listdir(PATH_DIR_CKPT)
h5s.sort()
saved_h5 = os.path.join(PATH_DIR_CKPT, h5s[-1])
m.construct_model(saved_h5, weights_only=True)
else:
# create new model
msg = "{} doesn't exist or is empty. Creating new model."
print(bc.WARNING + bc.BOLD + msg.format(PATH_DIR_SAVE) + bc.ENDC)
os.makedirs(PATH_DIR_CKPT, exist_ok=True)
m.construct_model()
m.train_model(gen_tr,
gen_va,
n_epoch=n_epoch,
callbacks=callbacks,
verbose=False,
workers=args.w,
use_mp=True,
max_q_size=args.q,
initial_epoch=initial_epoch)
'hl_activations': [ReLU, ReLU, ReLU, ReLU],
'hl_sizes': [512, 512, 512, 256],
'decay': 0.,
'bNorm': True,
'dropout': True,
'regularizer': None
}
X_tr, Y_tr = load_data(args.datapath, args.agent, num_games=10)
X_val, Y_val = load_data(args.datapath, args.agent, num_games=30)
m = Mlp(io_sizes=(658, 20),
out_activation=Softmax,
loss='categorical_crossentropy',
metrics=['accuracy'],
**hypers,
verbose=0)
m.construct_model()
m.hist = m.model.fit(X_tr,
Y_tr,
hypers['batch_size'],
args.epochs,
validation_data=(X_val, Y_val),
verbose=0)
val_acc_str = str(m.hist.history['val_accuracy'][-1])[:6]
print("Agent %s got val acc: %s" % (args.agent, val_acc_str))
savepath = os.path.join(args.savedir, args.agent, val_acc_str + '.h5')
m.model.save_weights(savepath)