def get_seed_file(data_folder, parameter_dictionary):
"""
Get seed file that matches a particular parameter dictionary
"""
# Get pre and post seed params for model
#parameter_list = ["centralised_rwg"] + parameter_list[2:-7]
copied_dictionary = deepcopy(parameter_dictionary)
copied_dictionary["general"]["learning_type"] = "centralised"
copied_dictionary["general"]["algorithm_selected"] = "rwg"
# If individual reward, allow seeds that used any number of agents
if copied_dictionary["general"]["reward_level"] == "individual":
copied_dictionary["environment"]["slope"]["num_agents"] = 1
# Get list of all seed files with matching parameters
seedfile_prefix = "_".join(
str(item)
for item in Learner.get_core_params_in_model_name(copied_dictionary))
seedfile_extension = ".npy"
possible_seedfiles = glob(
f'{data_folder}/{seedfile_prefix}*{seedfile_extension}')
# Return seed_file name but throw error if there's more than one
if len(possible_seedfiles) == 0:
raise RuntimeError('No valid seed files')
elif len(possible_seedfiles) > 1:
raise RuntimeError('Too many valid seed files')
else:
return possible_seedfiles[0]
def get_seed_genome(self):
"""
Get the seed genome to be used as cma's mean. If getting it from a file, the appropriate seed genome is the one
that has the same parameter values as the current experiment
@return: Genome and its fitness
"""
if self.parameter_dictionary['algorithm']['cma']['seeding_required'] == "True":
dictionary_copy = copy.deepcopy(self.parameter_dictionary)
if dictionary_copy['algorithm']['cma']['partial'] == "False":
dictionary_copy['general']['algorithm_selected'] = "rwg"
elif dictionary_copy['algorithm']['cma']['partial'] == "True":
dictionary_copy['general']['algorithm_selected'] = "cma"
else:
raise RuntimeError("The value for partial cma is neither True nor False")
# If individual reward, look for seeds that use just one agent
# TODO: Should the agent learn in an environment with multiple other agents?
if dictionary_copy['general']['reward_level'] == "individual":
environment_name = dictionary_copy['general']['environment']
dictionary_copy['environment'][environment_name]['num_agents'] = '1'
# If this is a decentralised cma learner, use a centralised rwg seed
if dictionary_copy['general']['learning_type'] == "decentralised":
dictionary_copy['general']['learning_type'] = "centralised"
# Looks for seedfiles with the same parameters as the current experiment
parameters_in_name = Learner.get_core_params_in_model_name(dictionary_copy)
possible_seedfiles = None
seedfile_prefix = "_".join([str(param) for param in parameters_in_name])
seedfile_extension = self.Agent.get_model_file_extension()
possible_seedfiles = glob(f'{seedfile_prefix}*{seedfile_extension}')
# Makes sure there is only one unambiguous seedfile
if len(possible_seedfiles) == 0:
raise RuntimeError('No valid seed files')
elif len(possible_seedfiles) > 1:
raise RuntimeError('Too many valid seed files')
else:
model_file_extension = self.Agent.get_model_file_extension()
seed_fitness = float(possible_seedfiles[0].split("_")[-1].strip(model_file_extension))
return self.Agent.load_model_from_file(possible_seedfiles[0]), seed_fitness
else:
mean = 0
std = 1
seed = self.parameter_dictionary['general']['seed']
mean_array = [mean] * self.genome_length
random_variable = multivariate_normal(mean=mean_array, cov=np.identity(self.genome_length) * std)
seed_genome = random_variable.rvs(1, seed)
seed_fitness = None
return seed_genome, seed_fitness
def get_best_of_k_samples(results_file, parameter_filename, k, N_episodes):
"""
From a results file with n samples. Find the best among the first k samples and write it to a file
@param results_file: CSV file containing n models
@param parameter_filename: Parameter file used to generate results
@param k: An integer
@param N_episodes: Number of episodes used to evaluate the sample
"""
# Load json and csv files
parameter_dictionary = json.loads(open(parameter_filename).read())
f = open(results_file, "r")
data = f.read().strip().split("\n")
# Get best genome in first k samples
best_genome = np.array(
[float(element) for element in data[0].split(",")[0:-N_episodes]])
best_score = np.mean([
float(episode_score)
for episode_score in data[0].split(",")[-N_episodes:]
])
for row in data[0:k]:
genome = np.array(
[float(element) for element in row.split(",")[0:-N_episodes]])
episode_scores = [
float(score) for score in row.split(",")[-N_episodes:]
]
mean_score = np.mean(episode_scores)
if mean_score > best_score:
best_score = mean_score
best_genome = genome
# Generate model name
if parameter_dictionary['general']['reward_level'] == "team":
parameter_dictionary['algorithm']['agent_population_size'] = k * 2
else:
parameter_dictionary['algorithm']['agent_population_size'] = k
parameters_in_name = Learner.get_core_params_in_model_name(
parameter_dictionary)
parameters_in_name += CentralisedRWGLearner.get_additional_params_in_model_name(
parameter_dictionary)
parameters_in_name += [best_score]
model_name = "_".join([str(param) for param in parameters_in_name])
np.save(model_name, best_genome)
def get_results_headings(parameter_dictionary):
"""
Get a list containing (most) of the columns that will be printed to the results file
@param parameter_dictionary: Dictionary containing parameter values
@return: List of column names
"""
headings = Learner.get_results_headings(parameter_dictionary)
headings += [
"agent_population_size", "generations", "mutation_probability",
"tournament_size", "mu", "sigma", "num_parents", "num_children",
"crowding_factor"
]
return headings
def get_results_headings(parameter_dictionary):
"""
Get a list containing (most) of the columns that will be printed to the results file
@param parameter_dictionary: Dictionary containing parameter values
@return: List of column names
"""
headings = Learner.get_results_headings(parameter_dictionary)
headings += ["agent_population_size",
"sigma",
"generations",
"tolx",
"tolfunhist",
"tolflatfitness"
"tolfun",
"tolstagnation"]
return headings
def generate_rwg_experiments(experiment_directory, core_parameter_filename,
learning_type, reward_level, list_file_name,
num_agents_in_setup, num_seeds_for_team,
generator_seed, many_architectures):
num_agents_in_setup = [
int(num) for num in num_agents_in_setup.strip('[]').split(',')
]
num_seeds_for_team = int(num_seeds_for_team)
generator_seed = int(generator_seed)
if learning_type == "decentralised":
raise RuntimeError("No support for decentralised rwg yet")
for num_agents in num_agents_in_setup:
parameter_dictionary = json.loads(open(core_parameter_filename).read())
environment_name = parameter_dictionary["general"]["environment"]
parameter_dictionary["general"]["reward_level"] = reward_level
parameter_dictionary["environment"][environment_name][
"num_agents"] = num_agents
#TODO: Modify this for constant arena size
if environment_name == "slope":
parameter_dictionary["environment"][environment_name][
"arena_width"] = num_agents * 2
parameter_dictionary["environment"][environment_name][
"num_resources"] = num_agents * 2
parameter_dictionary["algorithm"]["agent_population_size"] *= (
num_agents // 2)
parameter_dictionary["general"]["learning_type"] = learning_type
#TODO: WARNING! CHANGE WHEN NOT DOING FC
if learning_type == "fully-centralised":
default_hidden_units = 8
else:
default_hidden_units = parameter_dictionary["agent"]["nn"][
"hidden_units_per_layer"]
num_experiments = num_seeds_for_team
np_random = np.random.RandomState(generator_seed)
g = open(f"{experiment_directory}/data/{list_file_name}", "a")
for i in range(num_experiments):
new_seed = np_random.randint(low=1, high=2**32 - 1)
parameter_dictionary["general"]["seed"] = new_seed
for architecture in ["rnn", "ffnn"]:
for bias in ["True", "False"]:
for layers in [0, 1, 2]:
parameter_dictionary["agent"]["nn"][
"architecture"] = architecture
parameter_dictionary["agent"]["nn"]["bias"] = bias
parameter_dictionary["agent"]["nn"][
"hidden_layers"] = layers
if layers == 0:
parameter_dictionary["agent"]["nn"][
"hidden_units_per_layer"] = 0
else:
parameter_dictionary["agent"]["nn"][
"hidden_units_per_layer"] = default_hidden_units
parameters_in_filename = []
if not many_architectures:
parameters_in_filename += Learner.get_core_params_in_model_name(
parameter_dictionary)
if learning_type == "centralised":
parameters_in_filename += CentralisedRWGLearner.get_additional_params_in_model_name(
parameter_dictionary)
elif learning_type == "fully-centralised":
parameters_in_filename += FullyCentralisedRWGLearner.get_additional_params_in_model_name(
parameter_dictionary)
else:
if learning_type == "centralised":
parameters_in_filename += ["CTDE"]
elif learning_type == "fully-centralised":
parameters_in_filename += ["FC"]
parameters_in_filename += [
architecture, bias, layers
]
filename = "_".join(
[str(param)
for param in parameters_in_filename]) + ".json"
f = open(f"{experiment_directory}/data/{filename}",
"w")
dictionary_string = json.dumps(parameter_dictionary,
indent=4)
f.write(dictionary_string)
f.close()
g.write(
f"python3 experiment.py --parameters {filename}\n")
g.close()
def generate_ga_experiments(experiment_directory, core_parameter_filename,
learning_type, reward_level, list_file_name,
num_agents_in_setup, holding_constant,
num_seeds_for_team, generator_seed):
num_agents_in_setup = [
int(num) for num in num_agents_in_setup.strip('[]').split(',')
]
num_seeds_for_team = int(num_seeds_for_team)
generator_seed = int(generator_seed)
if holding_constant == "games_per_run":
pop_size_for_team = {
"centralised": {
2: 240,
4: 480,
6: 720,
8: 960,
10: 1200
},
"decentralised": {
2: 120,
4: 120,
6: 120,
8: 120,
10: 120
}
}
elif holding_constant == "games_per_learner":
pop_size_for_team = {
"centralised": {
2: 100,
4: 200,
6: 300,
8: 400,
10: 500
},
"decentralised": {
2: 100,
4: 200,
6: 300,
8: 400,
10: 500
},
"fully-centralised": {
2: 100,
4: 200,
6: 300,
8: 400,
10: 500
}
}
elif holding_constant == "games_per_run_2":
pop_size_for_team = {"centralised": {2: 200, 4: 800, 6: 1800}}
for num_agents in num_agents_in_setup:
parameter_dictionary = json.loads(open(core_parameter_filename).read())
environment_name = parameter_dictionary["general"]["environment"]
parameter_dictionary["general"]["reward_level"] = reward_level
parameter_dictionary["environment"][environment_name][
"num_agents"] = num_agents
# Comment out for constant arena size
'''if environment_name == "slope":
parameter_dictionary["environment"][environment_name]["arena_width"] = num_agents * 2
parameter_dictionary["environment"][environment_name]["num_resources"] = num_agents * 2'''
parameter_dictionary["algorithm"][
"agent_population_size"] = pop_size_for_team[learning_type][
num_agents]
parameter_dictionary["general"]["learning_type"] = learning_type
num_experiments = num_seeds_for_team
np_random = np.random.RandomState(generator_seed)
g = open(f"{experiment_directory}/data/{list_file_name}", "a")
for i in range(num_experiments):
new_seed = np_random.randint(low=1, high=2**32 - 1)
parameter_dictionary["general"]["seed"] = new_seed
parameters_in_filename = []
parameters_in_filename += Learner.get_core_params_in_model_name(
parameter_dictionary)
parameters_in_filename += GALearner.get_additional_params_in_model_name(
parameter_dictionary)
filename = "_".join(
[str(param) for param in parameters_in_filename]) + ".json"
f = open(f"{experiment_directory}/data/{filename}", "w")
dictionary_string = json.dumps(parameter_dictionary, indent=4)
f.write(dictionary_string)
f.close()
g.write(f"python3 experiment.py --parameters {filename}\n")
g.close()
def run_experiment(parameter_filename):
parameter_dictionary = json.loads(open(parameter_filename).read())
fitness_calculator = FitnessCalculator(parameter_filename)
if parameter_dictionary["general"]["algorithm_selected"] == "rwg":
if parameter_dictionary["general"]["learning_type"] == "centralised":
learner = CentralisedRWGLearner(fitness_calculator)
elif parameter_dictionary["general"]["learning_type"] == "fully-centralised":
learner = FullyCentralisedRWGLearner(fitness_calculator)
else:
raise RuntimeError("Invalid learning type for rwg")
genome, fitness = learner.learn()
elif parameter_dictionary["general"]["algorithm_selected"] == "cma":
if parameter_dictionary["algorithm"]["cma"]["seeding_included"] == "True":
#if parameter_dictionary["general"]["learning_type"] == "fully-centralised":
# raise RuntimeError("Seeding included is not yet supported for fully centralised")
# Load default rwg parameters
if parameter_dictionary["general"]["environment"] == "slope":
default_rwg_parameter_filename = 'default_slope_rwg_parameters_individual.json'
rwg_parameter_dictionary = json.loads(open(default_rwg_parameter_filename).read())
# Copy general parameters from cma to rwg
rwg_parameter_dictionary["general"] = copy.deepcopy(parameter_dictionary["general"])
rwg_parameter_dictionary["general"]["algorithm_selected"] = "rwg"
if parameter_dictionary["general"]["learning_type"] == "decentralised":
rwg_parameter_dictionary["general"]["learning_type"] = "centralised"
if parameter_dictionary["general"]["reward_level"] == "team":
rwg_parameter_dictionary["general"]["reward_level"] = "team"
environment_name = parameter_dictionary["general"]["environment"]
if parameter_dictionary["general"]["team_type"] == "heterogeneous":
rwg_parameter_dictionary["algorithm"]["agent_population_size"] *= parameter_dictionary["environment"][environment_name]["num_agents"]
# Copy environment parameters from cma to rwg
rwg_parameter_dictionary["environment"] = copy.deepcopy(parameter_dictionary["environment"])
if rwg_parameter_dictionary["general"]["reward_level"] == "individual":
environment_name = rwg_parameter_dictionary["general"]["environment"]
rwg_parameter_dictionary["environment"][environment_name]["num_agents"] = 1
# Copy agent parameters from cma to rwg
rwg_parameter_dictionary["agent"] = copy.deepcopy(parameter_dictionary["agent"])
# Create rwg json file and load to fitness calculator
parameters_in_name = Learner.get_core_params_in_model_name(rwg_parameter_dictionary)
if parameter_dictionary["general"]["learning_type"] == "centralised":
parameters_in_name += CentralisedRWGLearner.get_additional_params_in_model_name(rwg_parameter_dictionary)
elif parameter_dictionary["general"]["learning_type"] == "fully-centralised":
parameters_in_name += FullyCentralisedRWGLearner.get_additional_params_in_model_name(rwg_parameter_dictionary)
new_rwg_parameter_filename = "_".join([str(param) for param in parameters_in_name]) + ".json"
f = open(new_rwg_parameter_filename, "w")
rwg_dictionary_string = json.dumps(rwg_parameter_dictionary, indent=4)
f.write(rwg_dictionary_string)
f.close()
rwg_fitness_calculator = FitnessCalculator(new_rwg_parameter_filename)
# Seeding
if parameter_dictionary["general"]["learning_type"] == "fully-centralised":
learner1 = FullyCentralisedRWGLearner(rwg_fitness_calculator)
else:
learner1 = CentralisedRWGLearner(rwg_fitness_calculator)
genome1, fitness1 = learner1.learn()
# Learning
if parameter_dictionary["general"]["learning_type"] == "centralised":
learner2 = CentralisedCMALearner(fitness_calculator)
genome2, fitness2 = learner2.learn()
elif parameter_dictionary["general"]["learning_type"] == "decentralised":
learner2 = DecentralisedCMALearner(fitness_calculator)
genomes, fitnesses = learner2.learn()
elif parameter_dictionary["general"]["learning_type"] == "fully-centralised":
learner2 = FullyCentralisedCMALearner(fitness_calculator)
genomes, fitnesses = learner2.learn()
else:
if parameter_dictionary["general"]["learning_type"] == "centralised":
learner = CentralisedCMALearner(fitness_calculator)
genome, fitness = learner.learn()
elif parameter_dictionary["general"]["learning_type"] == "decentralised":
learner = DecentralisedCMALearner(fitness_calculator)
genomes, fitnesses = learner.learn()
elif parameter_dictionary["general"]["learning_type"] == "fully-centralised":
learner = FullyCentralisedCMALearner(fitness_calculator)
genomes, fitnesses = learner.learn()
elif parameter_dictionary["general"]["algorithm_selected"] == "ga":
if parameter_dictionary["general"]["learning_type"] == "centralised":
learner = CentralisedGALearner(fitness_calculator)
genome, fitness = learner.learn()
def get_incomplete_runs(old_directory, new_directory):
# For each cma json file
old_experiments_directory = os.path.join(old_directory, 'experiments')
old_results_directory = os.path.join(old_directory, 'results')
new_experiments_directory = os.path.join(new_directory, 'experiments')
cma_json_files = glob(f'{old_experiments_directory}/cma*json')
new_experiments_list = os.path.join(new_experiments_directory, "LIST_cma")
g = open(new_experiments_list, "a")
for cma_json in cma_json_files:
# If it doesn't have a final model
model_prefix = cma_json.split("/")[-1].strip(".json")
final_models = glob(
f"{old_results_directory}/{model_prefix}*_final.npy")
if len(final_models) > 1:
raise RuntimeError("Too many matching final models")
elif len(final_models) < 1:
# Find model with highest generation
other_generations = glob(
f"{old_results_directory}/{model_prefix}*.npy")
latest_model = None
latest_generation = 0
for model_name in other_generations:
generation = int(
model_name.split("/")[-1].strip(".npy").split("_")[-1])
if generation > latest_generation:
latest_generation = generation
latest_model = model_name
# Create new version of json with 'partialcma' in filename and dictionary
if latest_model is None:
raise RuntimeError(
"No saved models. Evolution didn't make it to 20 generations"
)
else:
# Create modified parameter dictionary for new run
parameter_dictionary = json.loads(open(cma_json).read())
parameter_dictionary["general"][
"algorithm_selected"] = "partialcma"
parameter_dictionary["algorithm"]["cma"][
"generations"] -= latest_generation
parameters_in_filename = []
parameters_in_filename += Learner.get_core_params_in_model_name(
parameter_dictionary)
parameters_in_filename += CMALearner.get_additional_params_in_model_name(
parameter_dictionary)
# Turn parameter dictionary into json file in new directory
filename = "_".join(
[str(param) for param in parameters_in_filename]) + ".json"
new_parameter_file = os.path.join(new_experiments_directory,
filename)
f = open(new_parameter_file, "w")
dictionary_string = json.dumps(parameter_dictionary, indent=4)
f.write(dictionary_string)
f.close()
# Add experiment to new LIST file
g.write(f"python3 experiment.py --parameters {filename}\n")
# Copy model to new directory
filename = os.path.join(new_experiments_directory,
latest_model.split("/")[-1])
copyfile(latest_model, filename)
g.close()