def main():
'''Main script function.'''
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.ERROR)
parser = argparse.ArgumentParser()
parser.add_argument("path", help="The path to save the results.")
parser.add_argument("--alg", help="The algorithm to use.",
default='MADDPG', type=str.upper)
parser.add_argument("--env_name", help="The gamma to use.",
default='simple_spread')
parser.add_argument('--total_timesteps', default=int(1e6), type=int,
help="Number of timesteps per training session")
parser.add_argument('--trials', help="The number of trials to run.",
default=10, type=int)
args = parser.parse_args()
parameters = vars(args).copy()
del parameters['trials']
path = Path(parameters['path'])
if not path.exists():
path.mkdir()
utils_file.save_json(parameters, path / 'parameters.json')
else:
if (path / 'study.db').exists():
print('Directory exists. Using existing study and parameters.')
parameters = utils_file.load_json(path / 'parameters.json')
else:
raise FileExistsError(('Directory already exists and is not a '
'study.'))
objective = partial(run_experiment, parameters)
storage = 'sqlite:///' + str(path / 'study.db')
study = optuna.create_study(study_name=str(path.name),
storage=storage, load_if_exists=True,
pruner=optuna.pruners.MedianPruner())
study.optimize(objective, n_trials=args.trials)
def run_experiment(parameters):
'''Set up and run an experiment.'''
repository_path = Path(__file__).parent
save_to = parameters['path']
task_name = '{alg}-{learning_rate:.4f}-{gamma:.4f}-{seed:d}'
task_name = task_name.format(**parameters)
save_to = Path(save_to, task_name)
with utils_file.create_directory(save_to, False, False) as log_dir:
parameters['commit'] = utils_file.get_commit_hash(repository_path)
utils_file.save_json(parameters, log_dir / 'hyperparams.json')
parameters['model_path'] = str(log_dir / 'model.pkl')
log_path = str(log_dir / 'monitor_{:d}')
env_name = parameters['env_name']
kwargs = parameters.setdefault('kwargs', {})
env_callable = [
partial(Monitor,
gym.make(env_name, **kwargs),
log_path.format(i),
allow_early_resets=True,
info_keywords=('loss', 'accuracy', 'actions_mean',
'weights_mean', 'actions_std',
'states_mean', 'grads_mean'),
chunk_size=parameters.setdefault('chunk_size', 5))
for i in range(1)
]
try:
run_agent(env_callable, parameters)
except RuntimeError as error:
LOGGER.error('%s, %s', error, parameters)
return parameters
def test_save_json(save_path):
'''Test save_json function.'''
data = {chr(97 + i): i for i in range(26)}
path = save_path / 'test_save_json.json'
utils_file.save_json(data, path)
dictionary = utils_file.load_json(path)
for i, (name, value) in enumerate(dictionary.items()):
assert i == value
assert name == chr(97 + i)
def main():
'''Main script function.'''
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.ERROR)
parser = argparse.ArgumentParser()
parser.add_argument("path", help="The path to save the results.")
parser.add_argument("--alg", help="The algorithm to use.", default='PPO')
parser.add_argument("--env_name", help="The gamma to use.",
default='MultiOptLRs-v0')
parser.add_argument('--total_timesteps', default=int(1e5), type=int,
help="Number of timesteps per training session")
parser.add_argument('--data_set', help="The data set to use.",
default='iris')
parser.add_argument('--trials', help="The number of trials to run.",
default=10, type=int)
args = parser.parse_args()
parameters = vars(args).copy()
del parameters['trials']
path = Path(parameters['path'])
if not path.exists():
path.mkdir()
parameters['kwargs'] = {'data_set': parameters['data_set']}
utils_file.save_json(parameters, path / 'parameters.json')
else:
if (path / 'study.db').exists():
print('Directory exists. Using existing study and parameters.')
parameters = utils_file.load_json(path / 'parameters.json')
else:
raise FileExistsError(('Directory already exists and is not a '
'study.'))
storage = 'sqlite:///' + str(path / 'study.db')
study = optuna.create_study(
study_name=str(path.name),
storage=storage, load_if_exists=True,
pruner=optuna.pruners.MedianPruner()
)
print(study.best_value)
print(study.best_trial.number)
print(study.best_params)
#params = study.best_params
param_file = path / str(study.best_trial.number) / 'parameters.json'
params = utils_file.load_json(param_file)
kwargs = params['kwargs']
parameters.update({
'path': path / str(study.best_trial.number),
'gamma': params['gamma'],
'learning_rate': params['learning_rate'],
'kwargs': {
#'batch_size': int(params['batch_size']),
'max_batches': int(kwargs['max_batches']),
'max_history': int(kwargs['max_history']),
'max_batches': 1e3
}
})
live_plot(run_experiment(parameters), 'test')
def run_experiment(parameters, trial):
'''Set up and run an experiment.'''
parameters = parameters.copy()
batch_size = [2**i for i in range(5, 12)]
batch_size = int(trial.suggest_categorical('batch_size', batch_size))
obs_version = int(trial.suggest_int('observation_version', 0, 3))
max_history = int(trial.suggest_discrete_uniform('max_history', 5, 50, 5))
learning_rate = float(trial.suggest_loguniform('learning_rate', 1e-5, 1e0))
buffer_size = int(trial.suggest_int('replay_buffer_size', 1e3, 1e5))
parameters.update({
'gamma': float(trial.suggest_uniform('gamma', 0.1, 1.0)),
'replay_buffer_size': buffer_size
})
parameters.setdefault('agent_parameters', {}).update({
'policy': {
'learning_rate': learning_rate
},
'critic': {
'learning_rate': learning_rate
}
})
parameters.setdefault('kwargs', {}).update({
'batch_size':
batch_size,
'version':
int(trial.suggest_int('version', 0, 5)),
'observation_version':
obs_version,
'max_history':
max_history,
'reward_version':
int(trial.suggest_int('reward_version', 0, 1)),
'action_version':
int(trial.suggest_int('action_version', 0, 1))
})
path = Path(parameters['path']) / str(trial.number)
path.mkdir()
parameters['path'] = str(path)
parameters['commit'] = utils_file.get_commit_hash(Path(__file__).parent)
utils_file.save_json(parameters, path / 'parameters.json')
return run_agents(parameters, trial)
def run_experiment(parameters, trial):
'''Set up and run an experiment.'''
parameters = parameters.copy()
parameters.update(get_parameters(trial))
trial_path = Path(parameters['path'], str(trial.number))
trial_path.mkdir()
parameters['path'] = str(trial_path)
parameters['commit'] = utils_file.get_commit_hash(Path(__file__).parent)
log_path = str(trial_path / 'monitor_{:d}')
kwargs = parameters.setdefault('kwargs', {})
utils_file.save_json(parameters, trial_path / 'parameters.json')
wrapped_envs = [
partial(Monitor,
partial(gym.make, parameters['env_name'], **kwargs),
log_path.format(i),
info_keywords=('loss', 'actions_mean', 'weights_mean',
'actions_std', 'states_mean', 'grads_mean'),
chunk_size=parameters.setdefault('chunk_size', 5))
for i in range(1)
]
return -run_agent(wrapped_envs, parameters, trial)
def main():
'''Evaluate a trained model against logistic regression.'''
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.ERROR)
parser = argparse.ArgumentParser()
parser.add_argument("path", help="The path to save the results.")
parser.add_argument("--trials",
help="The number of trials.",
type=int,
default=10)
parser.add_argument("--total_epochs",
help="The number of epochs.",
type=int,
default=40)
parser.add_argument("--data_set",
help="The data set to trial against.",
type=str,
default='iris')
args = parser.parse_args()
parameters = vars(args).copy()
del parameters['trials']
path = Path(parameters['path'])
if not path.exists():
path.mkdir()
utils_file.save_json(parameters, path / 'parameters.json')
else:
if (path / 'study.db').exists():
print('Directory exists. Using existing study and parameters.')
parameters = utils_file.load_json(path / 'parameters.json')
else:
raise FileExistsError(('Directory already exists and is not a '
'study.'))
objective = partial(train_model, parameters)
storage = 'sqlite:///' + str(path / 'study.db')
study = optuna.create_study(study_name=str(path.name),
storage=storage,
load_if_exists=True,
pruner=optuna.pruners.MedianPruner())
study.optimize(objective, n_trials=args.trials)
def train_model(parameters, trial):
'''Train an model to classify a data set.'''
parameters = parameters.copy()
path = Path(parameters['path'], str(trial.number))
parameters['path'] = str(path)
learning_rate = trial.suggest_loguniform('learning_rate', 1e-5, 1e0)
parameters.update({'learning_rate': learning_rate})
sequence = load_data(parameters['data_set'])
layers = [sequence.feature_shape[0], 256, 256, sequence.target_shape[0]]
graph = tf.Graph()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.9
with graph.as_default():
tensor = model_in = tf.keras.layers.Input([layers[0]])
layers = [layers[0], layers[-1]]
for layer in layers[1:-1]:
layer = tf.keras.layers.Dense(layer, activation='relu')
tensor = layer(tensor)
layer = tf.keras.layers.Dense(layers[-1], activation='softmax')
tensor = layer(tensor)
model = tf.keras.Model(inputs=model_in, outputs=tensor)
callbacks = [CustomCallback(trial, 'loss')]
with tf.Session(graph=graph, config=config):
try:
model.compile(tf.train.AdamOptimizer(learning_rate),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit_generator(sequence,
epochs=parameters['total_epochs'],
verbose=0,
callbacks=callbacks)
finally:
path.mkdir()
utils_file.save_json(parameters, path / 'parameters.json')
dataframe = pd.DataFrame.from_dict(callbacks[0].history)
dataframe.to_csv(path / 'results.csv')
model.save(str(path / 'model.hdf5'))
return dataframe['loss'].iloc[-1]
def run_experiment(parameters, trial):
'''Set up and run an experiment.'''
parameters = parameters.copy()
learning_rate = float(trial.suggest_loguniform('learning_rate', 1e-5, 1e0))
buffer_size = int(trial.suggest_int('replay_buffer_size', 1e3, 1e5))
parameters.update({
'gamma': float(trial.suggest_uniform('gamma', 0.1, 1.0)),
'replay_buffer_size': buffer_size
})
parameters.setdefault('agent_parameters', {}).update({
'policy': {
'learning_rate': learning_rate
},
'critic': {
'learning_rate': learning_rate
}
})
path = Path(parameters['path']) / str(trial.number)
path.mkdir()
parameters['path'] = str(path)
parameters['commit'] = utils_file.get_commit_hash(Path(__file__).parent)
utils_file.save_json(parameters, path / 'parameters.json')
return run_agents(parameters, trial)