def main(file, use_horovod, gpus, config_path, checkpoint):
config = process_config(config_path, use_horovod, gpus, checkpoint)
setup_tf_config()
if os.path.exists(file):
image = Image.open(file).convert('L')
image = np.array(ImageOps.invert(image))
image = np.expand_dims(image, axis=2)
image = np.expand_dims(image, axis=0)
image = normalize_image(image)
data_loader = MNISTDataLoader(config)
model, _ = build_model_and_trainer(config, data_loader)
prediction = model.predict(image)
print(np.argmax(np.squeeze(prediction)))
else:
raise ValueError('File does not exist')
def main(use_horovod: bool, gpus: int, config_path: str,
checkpoint: int) -> None:
config = process_config(config_path, use_horovod, gpus, checkpoint)
is_master = setup_tf_config(config)
# copy source files
if is_master and not os.path.exists(config.exp.source_dir):
shutil.copytree(
os.path.abspath(os.path.curdir),
config.exp.source_dir,
ignore=lambda src, names:
{'datasets', '__pycache__', '.git', 'experiments', 'venv'})
data_loader = MNISTDataLoader(config)
_, trainer = build_model_and_trainer(config, data_loader)
print(f'Start Training Experiment {config.exp.name}')
trainer.train()
def main(use_horovod: bool, gpus: int, checkpoint: int,
config_path: str) -> None:
config = process_config(config_path, use_horovod, gpus, checkpoint)
# create tensorflow session and set as keras backed
tf_config = tf.ConfigProto()
if config.trainer.use_horovod:
import horovod.keras as hvd
hvd.init()
tf_config.gpu_options.allow_growth = True
tf_config.gpu_options.visible_device_list = str(hvd.local_rank())
is_master = not config.trainer.use_horovod
if not is_master:
import horovod.keras as hvd
is_master = hvd.rank() == 0
if is_master and not os.path.exists(config.exp.source_dir):
# copy source files
shutil.copytree(
os.path.abspath(os.path.curdir),
config.exp.source_dir,
ignore=lambda src, names:
{"datasets", "__pycache__", ".git", "experiments", "venv"})
tf_sess = tf.Session(config=tf_config)
K.set_session(tf_sess)
data_loader = get_data_loader(config=config)
model, trainer = build_model_and_trainer(config, data_loader)
print(f"Start Training Experiment {config.exp.name}")
try:
trainer.train()
except Exception as e:
send_noti_to_telegram(
f"an exception raised on training {config.exp.name}")
raise e
def main():
config = process_config('configs/config.json')
print(f"Experiment {config.exp.name}")
model, trainer = build_model_and_trainer(config)
print("Start Training")
trainer.train()