import gridworld
mdpFunction = getattr(gridworld, "get" + opts.grid)
mdp = mdpFunction()
mdp.setLivingReward(opts.livingReward)
mdp.setNoise(opts.noise)
env = gridworld.GridworldEnvironment(mdp)
###########################
# GET THE DISPLAY ADAPTER
###########################
import textGridworldDisplay
display = textGridworldDisplay.TextGridworldDisplay(mdp)
if not opts.textDisplay:
import graphicsGridworldDisplay
display = graphicsGridworldDisplay.GraphicsGridworldDisplay(
mdp, opts.gridSize, opts.speed)
try:
display.start()
except KeyboardInterrupt:
sys.exit(0)
###########################
# GET THE AGENT
###########################
import valueIterationAgents, qlearningAgents
a = None
if opts.agent == 'value':
a = valueIterationAgents.ValueIterationAgent(mdp, opts.discount,
opts.iters)
elif opts.agent == 'q':
def __init__(self, optimal_policy=None, expr_log_dir=None):
###########################
# GENERAL CONTROL
###########################
self.text_only = ExperimentConfigurator.experimentConfig['text_only']
self.display_speed = ExperimentConfigurator.experimentConfig[
'display_speed']
self.n_episodes = ExperimentConfigurator.gridWorldConfig['n_episodes']
self.discount = ExperimentConfigurator.gridWorldConfig['discount']
self.check_policy_converge = ExperimentConfigurator.experimentConfig[
'check_policy_converge']
self.optimal_policy = optimal_policy
self.expr_log_dir = expr_log_dir
self.save_VDBE = ExperimentConfigurator.experimentConfig['save_VDBE']
###########################
# GET THE INPUT MODULE
###########################
if ExperimentConfigurator.experimentConfig[
'agent_type'] == qlearningAgents.QLearningAgent.getAgentType():
self.user_input_module = None
else:
self.user_input_module = user_input.UserInputModule(
is_asyn=ExperimentConfigurator.TamerConfig['is_asyn_input'])
self.auto_feedback = AutoFeedback()
###########################
# GET THE GRIDWORLD
###########################
# noinspection PyUnresolvedReferences
import gridworld
mdp_function = getattr(
gridworld,
"get" + ExperimentConfigurator.gridWorldConfig['grid_name'])
self.mdp = mdp_function()
self.mdp.setLivingReward(
ExperimentConfigurator.gridWorldConfig['living_reward'])
self.mdp.setNoise(ExperimentConfigurator.gridWorldConfig['noise'])
self.env = gridworld.GridworldEnvironment(self.mdp)
###########################
# Variables used to store parameters values
###########################
# init VDBE values records
global VDBE_RECORDS
VDBE_RECORDS = dict()
for state in self.env.getGridWorld().getNonTerminalStates():
VDBE_RECORDS[state] = list()
###########################
# GET THE DISPLAY ADAPTER
###########################
import textGridworldDisplay
self.display = textGridworldDisplay.TextGridworldDisplay(self.mdp)
if not self.text_only:
import graphicsGridworldDisplay
self.display = graphicsGridworldDisplay.GraphicsGridworldDisplay(
self.mdp, ExperimentConfigurator.gridWorldConfig['grid_size'],
self.display_speed)
try:
self.display.start()
except KeyboardInterrupt:
sys.exit(0)
###########################
# GET THE TAMER AGENT
###########################
# env.getPossibleActions, opts.discount, opts.learningRate, opts.epsilon
# simulationFn = lambda agent, state: simulation.GridworldSimulation(agent,state,mdp)
self.gridWorldEnv = GridworldEnvironment(self.mdp)
action_function = lambda m_state: self.mdp.getPossibleActions(m_state)
q_learn_opts = {'actionFn': action_function}
if ExperimentConfigurator.experimentConfig[
'agent_type'] == qlearningAgents.QLearningAgent.getAgentType():
self.agent = qlearningAgents.QLearningAgent(**q_learn_opts)
elif ExperimentConfigurator.experimentConfig[
'agent_type'] == qlearningAgents.TamerQAgent.getAgentType():
self.agent = qlearningAgents.TamerQAgent(
max_n_experiences=ExperimentConfigurator.
TamerConfig['agent_max_n_experiences'],
window_size=ExperimentConfigurator.
TamerConfig['agent_window_size'],
is_asyn_input=ExperimentConfigurator.
TamerConfig['is_asyn_input'],
**q_learn_opts)
elif ExperimentConfigurator.experimentConfig[
'agent_type'] == preferenceTamerAgent.PreferenceTAMERAgent.getAgentType(
):
self.agent = preferenceTamerAgent.PreferenceTAMERAgent(
max_n_experiences=ExperimentConfigurator.
TamerConfig['agent_max_n_experiences'],
window_size=ExperimentConfigurator.
TamerConfig['agent_window_size'],
is_asyn_input=ExperimentConfigurator.
TamerConfig['is_asyn_input'],
**q_learn_opts)
def __init__(self, grid_name='DiscountGrid', discount=0.9, learning_rate=0.5, living_reward=0.0
, noise=0, epsilon=0.3, display_speed=0.5
, grid_size=150, text_only=False, n_episodes=100
, agent_window_size=1, agent_max_n_experiences=1000
, is_use_q_agent=False, is_asyn_input=True):
self.text_only = text_only
self.display_speed = display_speed
self.n_episodes = n_episodes
self.discount = discount
###########################
# GET THE INPUT MODULE
###########################
if is_use_q_agent:
self.user_input_module = None
else:
self.user_input_module = user_input.UserInputModule(is_asyn=is_asyn_input)
###########################
# GET THE GRIDWORLD
###########################
# noinspection PyUnresolvedReferences
import gridworld
mdp_function = getattr(gridworld, "get" + grid_name)
self.mdp = mdp_function()
self.mdp.setLivingReward(living_reward)
self.mdp.setNoise(noise)
self.env = gridworld.GridworldEnvironment(self.mdp)
###########################
# GET THE DISPLAY ADAPTER
###########################
import textGridworldDisplay
self.display = textGridworldDisplay.TextGridworldDisplay(self.mdp)
if not text_only:
import graphicsGridworldDisplay
self.display = graphicsGridworldDisplay.GraphicsGridworldDisplay(self.mdp, grid_size, display_speed)
try:
self.display.start()
except KeyboardInterrupt:
sys.exit(0)
###########################
# GET THE TAMER AGENT
###########################
import qlearningAgents
# env.getPossibleActions, opts.discount, opts.learningRate, opts.epsilon
# simulationFn = lambda agent, state: simulation.GridworldSimulation(agent,state,mdp)
self.gridWorldEnv = GridworldEnvironment(self.mdp)
action_function = lambda state: self.mdp.getPossibleActions(state)
q_learn_opts = {
'gamma': discount,
'alpha': learning_rate,
'epsilon': epsilon,
'actionFn': action_function
}
if is_use_q_agent:
self.agent = qlearningAgents.QLearningAgent(**q_learn_opts)
else:
self.agent = qlearningAgents.TamerQAgent(max_n_experiences=agent_max_n_experiences
, window_size=agent_window_size
, is_asyn_input=is_asyn_input
, **q_learn_opts)
def __init__(self, grid_name='DiscountGrid', discount=0.9, learning_rate=0.5, living_reward=0.0
, noise=0, epsilon=0.3, display_speed=0.5
, grid_size=150, text_only=False, n_episodes=100
, agent_window_size=1, agent_max_n_experiences=1000
, check_value_converge=False
, check_policy_converge=False
, optimal_policy=None
, expr_log_dir=None
, delta=0.02
, is_use_q_agent=False
, init_temp=1024.0
, temp_decrease_rate=2.0
, is_asyn_input=True):
###########################
# GENERAL CONTROL
###########################
self.text_only = text_only
self.display_speed = display_speed
self.n_episodes = n_episodes
self.discount = discount
self.check_value_converge = check_value_converge
self.check_policy_converge = check_policy_converge
self.optimal_policy = optimal_policy
self.expr_log_dir = expr_log_dir
self.delta = delta
self.save_VDBE = False
###########################
# GET THE INPUT MODULE
###########################
if is_use_q_agent:
self.user_input_module = None
else:
self.user_input_module = user_input.UserInputModule(is_asyn=is_asyn_input)
###########################
# GET THE GRIDWORLD
###########################
# noinspection PyUnresolvedReferences
import gridworld
mdp_function = getattr(gridworld, "get" + grid_name)
self.mdp = mdp_function()
self.mdp.setLivingReward(living_reward)
self.mdp.setNoise(noise)
self.env = gridworld.GridworldEnvironment(self.mdp)
###########################
# Variables used to store parameters values
###########################
# init VDBE values records
global VDBE_RECORDS
VDBE_RECORDS = dict()
for state in self.env.getGridWorld().getNonTerminalStates():
VDBE_RECORDS[state] = list()
###########################
# GET THE DISPLAY ADAPTER
###########################
import textGridworldDisplay
self.display = textGridworldDisplay.TextGridworldDisplay(self.mdp)
if not text_only:
import graphicsGridworldDisplay
self.display = graphicsGridworldDisplay.GraphicsGridworldDisplay(self.mdp, grid_size, display_speed)
try:
self.display.start()
except KeyboardInterrupt:
sys.exit(0)
###########################
# GET THE TAMER AGENT
###########################
import qlearningAgents
# env.getPossibleActions, opts.discount, opts.learningRate, opts.epsilon
# simulationFn = lambda agent, state: simulation.GridworldSimulation(agent,state,mdp)
self.gridWorldEnv = GridworldEnvironment(self.mdp)
action_function = lambda state: self.mdp.getPossibleActions(state)
q_learn_opts = {
'gamma': discount,
'alpha': learning_rate,
'epsilon': epsilon,
'actionFn': action_function,
'init_temp': init_temp,
'temp_decrease_rate': temp_decrease_rate
}
if is_use_q_agent:
self.agent = qlearningAgents.QLearningAgent(**q_learn_opts)
else:
self.agent = qlearningAgents.TamerQAgent(max_n_experiences=agent_max_n_experiences
, window_size=agent_window_size
, is_asyn_input=is_asyn_input
, **q_learn_opts)
def main(myargs):
sys.argv = myargs.split()
opts = parseOptions()
###########################
# GET THE GRIDWORLD
###########################
if opts.grid == 'VerticalBridgeGrid':
opts.gridSize = 120
import gridworld
mdpFunction = getattr(gridworld, "get" + opts.grid)
mdp = mdpFunction()
mdp.setLivingReward(opts.livingReward)
mdp.setNoise(opts.noise)
env = gridworld.GridworldEnvironment(mdp)
###########################
# GET THE DISPLAY ADAPTER
###########################
import textGridworldDisplay
display = textGridworldDisplay.TextGridworldDisplay(mdp)
if not opts.textDisplay:
import graphicsGridworldDisplay
display = graphicsGridworldDisplay.GraphicsGridworldDisplay(
mdp, opts.gridSize, opts.speed)
try:
display.start()
except KeyboardInterrupt:
sys.exit(0)
###########################
# GET THE AGENT
###########################
import valueIterationAgents, qlearningAgents
a = None
if opts.agent == 'value':
a = valueIterationAgents.ValueIterationAgent(mdp, opts.discount,
opts.iters)
elif opts.agent == 'q':
#env.getPossibleActions, opts.discount, opts.learningRate, opts.epsilon
#simulationFn = lambda agent, state: simulation.GridworldSimulation(agent,state,mdp)
gridWorldEnv = GridworldEnvironment(mdp)
actionFn = lambda state: mdp.getPossibleActions(state)
qLearnOpts = {
'gamma': opts.discount,
'alpha': opts.learningRate,
'epsilon': opts.epsilon,
'actionFn': actionFn
}
a = qlearningAgents.QLearningAgent(**qLearnOpts)
elif opts.agent == 'random':
# # No reason to use the random agent without episodes
if opts.episodes == 0:
opts.episodes = 10
class RandomAgent:
def getAction(self, state):
return random.choice(mdp.getPossibleActions(state))
def getValue(self, state):
return 0.0
def getQValue(self, state, action):
return 0.0
def getPolicy(self, state):
"NOTE: 'random' is a special policy value; don't use it in your code."
return 'random'
def update(self, state, action, nextState, reward):
pass
a = RandomAgent()
else:
if not opts.manual: raise 'Unknown agent type: ' + opts.agent
###########################
# RUN EPISODES
###########################
# DISPLAY Q/V VALUES BEFORE SIMULATION OF EPISODES
try:
if not opts.manual and opts.agent == 'value':
if opts.valueSteps:
for i in range(opts.iters):
tempAgent = valueIterationAgents.ValueIterationAgent(
mdp, opts.discount, i)
display.displayValues(tempAgent,
message="VALUES AFTER " + str(i) +
" ITERATIONS")
display.pause()
display.displayValues(a,
message="VALUES AFTER " + str(opts.iters) +
" ITERATIONS")
display.pause()
display.displayQValues(a,
message="Q-VALUES AFTER " +
str(opts.iters) + " ITERATIONS")
display.pause()
except KeyboardInterrupt:
sys.exit(0)
# FIGURE OUT WHAT TO DISPLAY EACH TIME STEP (IF ANYTHING)
displayCallback = lambda x: None
if not opts.quiet:
if opts.manual and opts.agent == None:
displayCallback = lambda state: display.displayNullValues(state)
else:
if opts.agent == 'random':
displayCallback = lambda state: display.displayValues(
a, state, "CURRENT VALUES")
if opts.agent == 'value':
displayCallback = lambda state: display.displayValues(
a, state, "CURRENT VALUES")
if opts.agent == 'q':
displayCallback = lambda state: display.displayQValues(
a, state, "CURRENT Q-VALUES")
messageCallback = lambda x: printString(x)
if opts.quiet:
messageCallback = lambda x: None
# FIGURE OUT WHETHER TO WAIT FOR A KEY PRESS AFTER EACH TIME STEP
pauseCallback = lambda: None
if opts.pause:
pauseCallback = lambda: display.pause()
# FIGURE OUT WHETHER THE USER WANTS MANUAL CONTROL (FOR DEBUGGING AND DEMOS)
if opts.manual:
decisionCallback = lambda state: getUserAction(state, mdp.
getPossibleActions)
else:
decisionCallback = a.getAction
# RUN EPISODES
if opts.episodes > 0:
print()
print("RUNNING", opts.episodes, "EPISODES")
print()
returns = 0
for episode in range(1, opts.episodes + 1):
returns += runEpisode(a, env, opts.discount, decisionCallback,
displayCallback, messageCallback, pauseCallback,
episode)
if opts.episodes > 0:
print()
print("AVERAGE RETURNS FROM START STATE: " +
str((returns + 0.0) / opts.episodes))
print()
print()
# DISPLAY POST-LEARNING VALUES / Q-VALUES
if opts.agent == 'q' and not opts.manual:
try:
display.displayQValues(a,
message="Q-VALUES AFTER " +
str(opts.episodes) + " EPISODES")
display.pause()
display.displayValues(a,
message="VALUES AFTER " +
str(opts.episodes) + " EPISODES")
display.pause()
except KeyboardInterrupt:
sys.exit(0)
