def main():
# creates an environment
env = gym.make("Pendulum-v0")
# create a mutation probability schedule
mut_sch = sch.ExponentialDecaySchedule(initial_prob=.2, decay_factor=1e-2)
# cross over probability schedule
cross_sch = sch.ConstantSchedule(0.7)
# random process for introducing noise
rand_proc = OrnsteinUhlenbeckProcess(theta=0.01)
# Evolution operators information
ev_conf = EvolutionConfig(sel_args={"k": 30, "tournsize": 3},
sel_func=tools.selTournament,
cross_args={"indpb": 0.2},
cross_func=tools.cxUniform,
mut_args={"mu": 0, "sigma": 0.1, "indpb": 0.1},
mut_func=tools.mutGaussian)
# GA configuration
ga_conf = GeneticAlgConfiguration(evol_config=ev_conf,
pop_size=30,
num_gens=10,
mf_tuning_range=[-0.1, 0.1],
lin_vars_file="res/pendulum_linvars.xml",
gft_file="res/pendulum.xml",
load_init_pop_file=None,
apply_evolution=True,
defuzz_method=dfz.centroid,
mutation_prob_schdl=mut_sch,
cross_prob_schdl=cross_sch,
learn_rb_ops=False)
# Sim execution configuration
sim_conf = SimExecutionConfiguration(env=env,
agents=[Agent(0, PendulumObs)],
max_time_steps=600,
episodes_per_ind=1,
noise_process=rand_proc,
action_space=Const.CONTINUOUS,
persist_cache_per_ind=False,
visualize_env=True)
runner = Runner(ga_config=ga_conf,
sim_config=sim_conf,
seed=5,
episode_finished_callback=episode_finished,
epoch_finished_callback=epoch_finished,
sim_finished_callback=sim_finished,
evolution_finished_callback=
lambda pop, m_prob, c_prob, epoch: mut_prob_series.addrecord(epoch, mut_sch.prob))
runner.run()
def main():
# creates an environment
env = gym.make("MountainCarContinuous-v0")
# chart series
weighted_avg = ana.WeightedAvg(beta=0.9)
all_ind_series = ana.Series(name="Individuals Performance")
avg_series = ana.Series(name="Average (window = {})".format(round((1 / (1 - weighted_avg.beta)))))
gen_series = ana.Series(name="Generation Performance")
mut_prob_series = ana.Series(name="Mutation probability")
# create linguistic variables in a registry
reg = xmlToLinvars(open(LIN_VARS_FILE).read())
# create GFT with linguistic variables in the registry
reg = xmlToGFT(open(GFT_FILE).read(), registry=reg, defuzz_method=dfz.centroid)
# create GA instance with the registry object
ga = GeneticAlgorithm(registry=reg, seed=5)
# create a mutation probability schedule
mut_sch = sch.ExponentialDecaySchedule(initial_prob=.1, decay_factor=1e-2)
# create GFT algorithm object with the registry
rand_proc = OrnsteinUhlenbeckProcess(theta=0.01)
alg = Algorithm(registry=reg, random_process=rand_proc)
# create a cache for managing simulation data
cache = Cache(reg.gft_dict.keys())
# get initial population
if LOAD_INIT_POP:
pop = ga.load_initial_population(QLFD_IND_FILE, POP_SIZE)
pop = pop[::-1]
print("Num. of loaded individuals =", len(pop))
else:
pop = ga.generate_initial_population(POP_SIZE)
# initialize epoch or generation counter
epoch = 0
# initialize individual counter
ind_count = 0
# create an object for retrieving input values
obs_accessor = MountainCarObs()
# perform the simulation for a specified number of generations
while epoch < NUM_OF_GENS:
# Run the simulation with the current population
for ind in pop:
ind_count += 1
# initialize reward accumulator for the individual
total_reward = 0
# configure the GFT with the current individual
alg.configuregft(chromosome=ind)
# control the environment with the configured GFT
# reset the environment
observation = env.reset()
# set the received observation as the current array for retrieving input values
obs_accessor.current_observation = observation
# run through the time steps of the simulation
for t in range(MAX_TIME_STEPS):
# show the environment
env.render()
# since only one agent applies to this case study set a dummy agent ID
agent_id = 0
# get an action
actions_dict, input_vec_dict = alg.executebfc(obs_accessor, agent_id, add_noise=True)
# mark the GFSs that executed for the agent in this time step
cache.mark(output_dict_keys=actions_dict.keys())
# apply the selected action to the environment and observe feedback
next_state, reward, done, _ = env.step(list(actions_dict.values()))
reward = reward_shaping(pos=next_state[0], r=reward)
# decompose the received reward
reward_dict = cache.decomposeReward(reward)
# create experiences for the agent with respect to each GFSs that executed for the agent
exp_dict = cache.createExperiences(agent_id=agent_id, action=list(actions_dict.values()),
dec_reward_dict=reward_dict,
input_vec_dict=input_vec_dict, output_dict=actions_dict,
next_state_dict=None)
# add the experiences of the agent to the cache
cache.addExperiences(time_step=t, exp_dict=exp_dict)
# set the received observation as the current array for retrieving input values
obs_accessor.current_observation = next_state
# accumulate the rewards of all time steps
total_reward += reward
# if the episode is over end the current episode
if done:
break
# save contents of the cache and clear it for the next episode
# cache.compute_states_value(gamma=.9)
cache.save_csv(path="data/")
print(
"Episode: {t}/{T} | score: {r}".format(t=ind_count, T=(NUM_OF_GENS * POP_SIZE),
r=total_reward))
# set the return from the environment as the fitness value of the current individual
ind.fitness.values = (total_reward,)
# save qualified individual
if SAVE_BEST and total_reward >= SCORE_THRESHOLD:
document = Document(name=QLFD_IND_FILE)
document.addline(line=Line().add(text=Text(str(ind))))
document.save(append=True)
# store the performance of this individual in the corresponding series
all_ind_series.addrecord(ind_count, total_reward)
weighted_avg.update(total_reward)
avg_series.addrecord(ind_count, weighted_avg.value)
# Logging and other I/O operations
print("Epoch {} completed".format(epoch))
record = ga.stats.compile(pop)
print("Statistics for epoch {} = {}".format(epoch, record))
ga.logbook.record(epoch=epoch, **record)
# store max return
gen_series.addrecord(epoch, record["max"])
if APPLY_EVO:
# perform evolution
offspring = applyEvolution(population=pop, ga_alg=ga, mut_sch=mut_sch, epoch=epoch)
# set offspring as current population
pop = offspring
# update mutation probability series
mut_prob_series.addrecord(epoch, mut_sch.prob)
# increment epoch
epoch += 1
# print logbook
ga.logbook.header = "epoch", "avg", "std", "min", "max"
print(ga.logbook)
# plotting
plot_charts(avg_series, mut_prob_series)
# terminates environment
env.close()
style.use("seaborn-paper")
# for reproducibility
np.random.seed(1)
random.seed(1)
tf.set_random_seed(1)
LIN_VARS_FILE = "cartpole_linvars.xml"
GFT_FILE = "cartpole_gft.xml"
model_path = "cartpole_model.h5"
MAX_NUM_EPISODES = 100
TIME_STEPS_BEFORE_TRAIN = 1000
epsilon = 1e-4
tau_sch = sch.ExponentialDecaySchedule(initial_prob=0.1, decay_factor=1e-1)
def main():
# creates an environment
env = gym.make("CartPole-v1")
# print observation space ranges
print("observation space ranges\nhigh = {}\nlow = {}\n".format(str(env.observation_space.high),
str(env.observation_space.low)))
# chart series
weighted_avg = ana.WeightedAvg(beta=0.9)
all_ind_series = ana.Series(name="Episode Performance")
avg_series = ana.Series(name="Average (window = {})".format(round((1 / (1 - weighted_avg.beta)))))
# create linguistic variables in a registry
def test_ExponentialDecaySchedule(self):
s = sch.ExponentialDecaySchedule(init_prob, decay_factor)
runschedule(s)