def run(config):
hvd.init()
# Threading configuration
if os.environ.get("OMP_NUM_THREADS", None) is not None:
logger.debug(f"OMP_NUM_THREADS is {os.environ.get('OMP_NUM_THREADS')}")
num_intra = int(os.environ.get("OMP_NUM_THREADS"))
tf.config.threading.set_intra_op_parallelism_threads(num_intra)
tf.config.threading.set_inter_op_parallelism_threads(2)
config["seed"]
seed = config["seed"]
if seed is not None:
np.random.seed(seed)
tf.random.set_seed(seed)
load_config(config)
# Scale batch size and learning rate according to the number of ranks
batch_size = config[a.hyperparameters][a.batch_size] * hvd.size()
learning_rate = config[a.hyperparameters][a.learning_rate] * hvd.size()
logger.info(
f"Scaled: 'batch_size' from {config[a.hyperparameters][a.batch_size]} to {batch_size} "
)
logger.info(
f"Scaled: 'learning_rate' from {config[a.hyperparameters][a.learning_rate]} to {learning_rate} "
)
config[a.hyperparameters][a.batch_size] = batch_size
config[a.hyperparameters][a.learning_rate] = learning_rate
input_shape, output_shape = setup_data(config)
search_space = setup_search_space(config, input_shape, output_shape, seed=seed)
# Initialize Horovod
model_created = False
try:
model = search_space.create_model()
model_created = True
except:
logger.info("Error: Model creation failed...")
logger.info(traceback.format_exc())
if model_created:
# Setup callbacks only
callbacks = [
# Horovod: broadcast initial variable states from rank 0 to all other processes.
# This is necessary to ensure consistent initialization of all workers when
# training is started with random weights or restored from a checkpoint.
hvd.callbacks.BroadcastGlobalVariablesCallback(0),
# Horovod: average metrics among workers at the end of every epoch.
#
# Note: This callback must be in the list before the ReduceLROnPlateau,
# TensorBoard or other metrics-based callbacks.
hvd.callbacks.MetricAverageCallback(),
# Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final
# accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during
# the first five epochs. See https://arxiv.org/abs/1706.02677 for details.
#! initial_lr argument is not available in horovod==0.19.0
hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=5, verbose=0),
]
cb_requires_valid = False # Callbacks requires validation data
callbacks_config = config[a.hyperparameters].get(a.callbacks, {})
if callbacks_config is not None:
for cb_name, cb_conf in callbacks_config.items():
if cb_name in default_callbacks_config:
# cb_bame in hvd_root_cb implies hvd.rank() == 0
if not (cb_name in hvd_root_cb) or hvd.rank() == 0:
default_callbacks_config[cb_name].update(cb_conf)
# Import and create corresponding callback
Callback = import_callback(cb_name)
callbacks.append(Callback(**default_callbacks_config[cb_name]))
if cb_name in ["EarlyStopping"]:
cb_requires_valid = "val" in cb_conf["monitor"].split("_")
else:
logger.error(f"'{cb_name}' is not an accepted callback!")
trainer = HorovodTrainerTrainValid(config=config, model=model)
trainer.callbacks.extend(callbacks)
last_only, with_pred = preproc_trainer(config)
last_only = last_only and not cb_requires_valid
history = trainer.train(with_pred=with_pred, last_only=last_only)
result = compute_objective(config["objective"], history)
else:
# penalising actions if model cannot be created
result = -1
if result < -10:
result = -10
return result
def run_distributed_base_trainer(config):
physical_devices = tf.config.list_physical_devices("GPU")
try:
for i in range(len(physical_devices)):
tf.config.experimental.set_memory_growth(physical_devices[i], True)
except:
# Invalid device or cannot modify virtual devices once initialized.
pass
distributed_strategy = tf.distribute.MirroredStrategy()
n_replicas = distributed_strategy.num_replicas_in_sync
seed = config["seed"]
if seed is not None:
np.random.seed(seed)
tf.random.set_seed(seed)
load_config(config)
# Scale batch size and learning rate according to the number of ranks
initial_lr = config[a.hyperparameters][a.learning_rate]
if config[a.hyperparameters].get("lsr_batch_size"):
batch_size = config[a.hyperparameters][a.batch_size] * n_replicas
else:
batch_size = config[a.hyperparameters][a.batch_size]
if config[a.hyperparameters].get("lsr_learning_rate"):
learning_rate = config[a.hyperparameters][a.learning_rate] * n_replicas
else:
learning_rate = config[a.hyperparameters][a.learning_rate]
logger.info(
f"Scaled: 'batch_size' from {config[a.hyperparameters][a.batch_size]} to {batch_size} "
)
logger.info(
f"Scaled: 'learning_rate' from {config[a.hyperparameters][a.learning_rate]} to {learning_rate} "
)
config[a.hyperparameters][a.batch_size] = batch_size
config[a.hyperparameters][a.learning_rate] = learning_rate
input_shape, output_shape = setup_data(config)
search_space = get_search_space(config,
input_shape,
output_shape,
seed=seed)
model_created = False
with distributed_strategy.scope():
try:
model = search_space.sample(config["arch_seq"])
model_created = True
except:
logger.info("Error: Model creation failed...")
logger.info(traceback.format_exc())
else:
# Setup callbacks
callbacks = []
cb_requires_valid = False # Callbacks requires validation data
callbacks_config = config["hyperparameters"].get("callbacks")
if callbacks_config is not None:
for cb_name, cb_conf in callbacks_config.items():
if cb_name in default_callbacks_config:
default_callbacks_config[cb_name].update(cb_conf)
# Special dynamic parameters for callbacks
if cb_name == "ModelCheckpoint":
default_callbacks_config[cb_name][
"filepath"] = f'best_model_{config["id"]}.h5'
# replace patience hyperparameter
if "patience" in default_callbacks_config[cb_name]:
patience = config["hyperparameters"].get(
f"patience_{cb_name}")
if patience is not None:
default_callbacks_config[cb_name][
"patience"] = patience
# Import and create corresponding callback
Callback = import_callback(cb_name)
callbacks.append(
Callback(**default_callbacks_config[cb_name]))
if cb_name in ["EarlyStopping"]:
cb_requires_valid = "val" in cb_conf[
"monitor"].split("_")
else:
logger.error(
f"'{cb_name}' is not an accepted callback!")
# WarmupLR
if config[a.hyperparameters].get("warmup_lr"):
warmup_epochs = config[a.hyperparameters].get(
"warmup_epochs", 5)
callbacks.append(
LearningRateWarmupCallback(
n_replicas=n_replicas,
warmup_epochs=warmup_epochs,
verbose=0,
initial_lr=initial_lr,
))
trainer = BaseTrainer(config=config, model=model)
trainer.callbacks.extend(callbacks)
last_only, with_pred = preproc_trainer(config)
last_only = last_only and not cb_requires_valid
if model_created:
history = trainer.train(with_pred=with_pred, last_only=last_only)
# save history
save_history(config.get("log_dir", None), history, config)
result = compute_objective(config["objective"], history)
else:
# penalising actions if model cannot be created
result = -1
if result < -10 or np.isnan(result):
result = -10
return result
def run(config) -> Dict[str, Any]:
# Threading configuration
if os.environ.get("OMP_NUM_THREADS", None) is not None:
logger.debug(f"OMP_NUM_THREADS is {os.environ.get('OMP_NUM_THREADS')}")
num_intra = int(os.environ.get("OMP_NUM_THREADS"))
tf.config.threading.set_intra_op_parallelism_threads(num_intra)
tf.config.threading.set_inter_op_parallelism_threads(2)
seed = config["seed"]
if seed is not None:
np.random.seed(seed)
tf.random.set_seed(seed)
print('cwd in run (alpha.py) --- ', config['cwd'])
os.chdir(config['cwd'])
load_config(config)
print(
f'config in alpha.py ------------------------------------- hyliu ---------------{socket.gethostname()} ------------------',
config)
input_shape, output_shape = setup_data(config)
search_space = setup_search_space(config,
input_shape,
output_shape,
seed=seed)
model_created = False
node_of_training = socket.gethostname()
# logger.info(f'alpha.py run ---- config {config["id"]}')
result_dict = {}
try:
model = search_space.create_model()
model_created = True
except:
logger.info("Error: Model creation failed...")
logger.error(traceback.format_exc())
print(f'Error: Model creation failed... {traceback.format_exc()}')
if model_created:
# print('hyliu ---------------- config -------------', config)
model_id = '___'.join([str(i) for i in config['arch_seq']])
# Setup callbacks
callbacks = []
cb_requires_valid = False # Callbacks requires validation data
callbacks_config = config["hyperparameters"].get("callbacks")
if callbacks_config is not None:
for cb_name, cb_conf in callbacks_config.items():
if cb_name in default_callbacks_config:
default_callbacks_config[cb_name].update(cb_conf)
# Special dynamic parameters for callbacks
if cb_name == "ModelCheckpoint":
default_callbacks_config[cb_name][
"filepath"] = f'best_model_{model_id}.h5'
# Import and create corresponding callback
Callback = import_callback(cb_name)
callbacks.append(
Callback(**default_callbacks_config[cb_name]))
if cb_name in ["EarlyStopping"]:
cb_requires_valid = "val" in cb_conf["monitor"].split(
"_")
else:
logger.error(f"'{cb_name}' is not an accepted callback!")
trainer = TrainerTrainValid(config=config, model=model)
trainer.callbacks.extend(callbacks)
last_only, with_pred = preproc_trainer(config)
last_only = last_only and not cb_requires_valid
print('where am I? in alpha.py ', os.getcwd())
history = trainer.train(with_pred=with_pred, last_only=last_only)
result = compute_objective(config["objective"], history)
result_dict['history'] = history
# print(f'result {result} in alpha.py')
else:
# penalising actions if model cannot be created
result = -1
if result < -10:
result = -10
result_dict['result'] = result
result_dict['node'] = node_of_training
return result_dict
def run(config: dict) -> float:
# Threading configuration
if os.environ.get("OMP_NUM_THREADS", None) is not None:
logger.debug(f"OMP_NUM_THREADS is {os.environ.get('OMP_NUM_THREADS')}")
num_intra = int(os.environ.get("OMP_NUM_THREADS"))
tf.config.threading.set_intra_op_parallelism_threads(num_intra)
tf.config.threading.set_inter_op_parallelism_threads(2)
seed = config["seed"]
if seed is not None:
np.random.seed(seed)
tf.random.set_seed(seed)
load_config(config)
input_shape, output_shape = setup_data(config)
search_space = setup_search_space(config,
input_shape,
output_shape,
seed=seed)
model_created = False
try:
model = search_space.create_model()
model_created = True
except:
logger.info("Error: Model creation failed...")
logger.info(traceback.format_exc())
if model_created:
# Setup callbacks
callbacks = []
cb_requires_valid = False # Callbacks requires validation data
callbacks_config = config["hyperparameters"].get("callbacks")
if callbacks_config is not None:
for cb_name, cb_conf in callbacks_config.items():
if cb_name in default_callbacks_config:
default_callbacks_config[cb_name].update(cb_conf)
# Special dynamic parameters for callbacks
if cb_name == "ModelCheckpoint":
default_callbacks_config[cb_name][
"filepath"] = f'best_model_{config["id"]}.h5'
# replace patience hyperparameter
if "patience" in default_callbacks_config[cb_name]:
patience = config["hyperparameters"].get(
f"patience_{cb_name}")
if patience is not None:
default_callbacks_config[cb_name][
"patience"] = patience
# Import and create corresponding callback
Callback = import_callback(cb_name)
callbacks.append(
Callback(**default_callbacks_config[cb_name]))
if cb_name in ["EarlyStopping"]:
cb_requires_valid = "val" in cb_conf["monitor"].split(
"_")
else:
logger.error(f"'{cb_name}' is not an accepted callback!")
trainer = TrainerTrainValid(config=config, model=model)
trainer.callbacks.extend(callbacks)
last_only, with_pred = preproc_trainer(config)
last_only = last_only and not cb_requires_valid
history = trainer.train(with_pred=with_pred, last_only=last_only)
# save history
save_history(config.get("log_dir", None), history, config)
result = compute_objective(config["objective"], history)
else:
# penalising actions if model cannot be created
result = -1
if result < -10 or np.isnan(result):
result = -10
return result