def test_suck(self):
s1 = vwmodel.VWState(np.array([2, 2]),
np.array([[1, 0, 1], [0, 0, 0], [1, 0, 1]]))
s2 = vwmodel.VWState(np.array([2, 2]),
np.array([[1, 0, 1], [0, 0, 0], [1, 0, 0]]))
a = vwmodel.VWSuckAction()
self.assertEqual(s2, a.apply(s1))
def test_move(self):
s1 = vwmodel.VWState(np.array([3, 2]),
np.array([[1, 0, 1], [0, 0, 0], [1, 0, 1]]))
s2 = vwmodel.VWState(np.array([2, 0]),
np.array([[1, 0, 1], [0, 0, 0], [1, 0, 1]]))
s3 = vwmodel.VWState(np.array([7, 5]),
np.array([[1, 0, 1], [0, 0, 0], [1, 0, 1]]))
a1 = vwmodel.VWMoveAction(np.array([-1, -2]))
a2 = vwmodel.VWMoveAction(np.array([5, 5]))
self.assertEqual(s2, a1.apply(s1))
self.assertEqual(s3, a2.apply(a1.apply(s1)))
def test_state(self):
s1 = vwmodel.VWState(np.array([3, 2]),
np.array([[1, 0, 1], [0, 0, 0], [1, 0, 1]]))
s2 = vwmodel.VWState(np.array([3, 1]),
np.array([[1, 0, 1], [0, 0, 0], [1, 0, 1]]))
s3 = vwmodel.VWState(np.array([3, 2]),
np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]]))
s4 = vwmodel.VWState(
np.array([3, 2]),
np.array([[1, 0, 1, 0], [0, 1, 0, 0], [1, 0, 1, 0]]))
self.assertNotEqual(s1, s2)
self.assertNotEqual(s2, s3)
self.assertNotEqual(s1, s3)
self.assertNotEqual(s1, s4)
self.assertEqual(s1, s1)
def test_is_legal(self):
s1 = vwmodel.VWState(np.array([0, 0]), np.array([[1, 0, 1], [0, 0,
0]]))
s2 = vwmodel.VWState(np.array([1, 1]), np.array([[1, 1, 1], [0, 0,
0]]))
s3 = vwmodel.VWState(np.array([0, 0]), np.array([[1, 1, 1], [0, 0,
0]]))
s4 = vwmodel.VWState(np.array([-1, 0]), np.array([[1, 0, 1], [1, 0,
0]]))
s5 = vwmodel.VWState(np.array([0, 1]), np.array([[1, 0, 1], [0, 0,
1]]))
m = self._model1
self.assertTrue(m.is_legal(s1))
self.assertFalse(m.is_legal(s2)) # invalid dust location
self.assertFalse(m.is_legal(s3)) # invalid dust location 2
self.assertFalse(m.is_legal(s4)) # invalid robot location
self.assertFalse(m.is_legal(s5)) # invalid robot location 2
def test_T(self):
# Test a move action. Probabilities calculated by hand.
q = self._dust_prob
p = self._act_fail
s = vwmodel.VWState(np.array([1, 1]), np.array([[1, 0, 0], [0, 1, 0]]))
a = vwmodel.VWMoveAction(np.array([0, 1]))
T = self._model1.T(s, a)
act_prob = {
(1, 2): (1 - p),
(1, 1): 2 * p / 3,
(1, 0): p / 3
} # probability of robot's location
for loc in [(1, 2), (1, 1), (1, 0)]:
for layout in util.functions.bitstrings(3):
inds = [(1, 0), (1, 2), (0, 2)]
dust = np.array(s.dust) # copy current state's dust
dust = util.functions.sparse_matrix((2, 3), inds, layout,
dust) # update dust
n_dust = np.sum(dust) # numbero of dust on the map
s_p = vwmodel.VWState(np.array(loc), dust)
dust_prob = (q**(n_dust - 2)) * (
(1 - q)**(5 - n_dust)) # probability of dust layout
self.assertTrue(
np.abs(T[s_p] - act_prob[loc] * dust_prob) < 1e-10)
# Test a suck Action
a = vwmodel.VWSuckAction()
T = self._model1.T(s, a)
for layout in util.functions.bitstrings(3):
inds = [(1, 0), (1, 2), (0, 2)]
dust = np.array(s.dust)
dust = util.functions.sparse_matrix((2, 3), inds, layout, dust)
n_dust = np.sum(dust)
dust[1, 1] = 0
s_p = vwmodel.VWState(s.robot, dust)
dust_prob = (q**(n_dust - 2)) * ((1 - q)**(5 - n_dust))
self.assertTrue(np.abs(T[s_p] - dust_prob) < 1e-10)
for s in self._model1.S():
for a in self._model1.A():
T = self._model1.T(s, a)
self.assertTrue(abs(sum(T.values()) - 1.0) < 10e-10)
def test_S(self):
s1 = self._map1.shape
m1 = self._model1
self.assertTrue(len(m1.S()) == 5 * 2**5)
for loc in np.transpose(np.nonzero(np.ones(s1))):
for layout in util.functions.bitstrings(np.prod(s1)):
inds = np.transpose(np.nonzero(np.ones(s1)))
s = vwmodel.VWState(
loc, util.functions.sparse_matrix(s1, inds, layout))
if m1.is_legal(s):
self.assertTrue(s in m1.S())
def test_reward(self):
reward = vwreward.VWReward()
dust = np.array([[1, 0], [1, 1]])
s = vwmodel.VWState(np.array([1, 1]), dust)
a = vwmodel.VWSuckAction()
self.assertEqual(reward.reward(s, a), -3)
a = vwmodel.VWMoveAction(np.array([-1, 0]))
self.assertEqual(reward.reward(s, a), -4)
def vacuum_main():
random.seed(0)
np.random.seed(0)
## Initialize constants
# map = np.array( [[1, 1, 1, 1],
# [1, 0, 1, 1],
# [1, 1, 0, 1]])
map = np.array( [[1, 1, 1],
[1, 0, 1]])
p_fail = 0.2
p_dust = 0.05
# start_state = vwmodel.VWState( np.array( [0,0] ),
# np.array( [[1, 1, 0, 0],
# [0, 0, 1, 0],
# [0, 0, 0, 0]] )
# )
start_state = vwmodel.VWState( np.array( [0,0] ),
np.array( [[1, 1, 0],
[0, 0, 1]] )
)
initial = util.classes.NumMap( {start_state:1.0} )
t_max = 500
## Initialize model
model = vwmodel.VWModel(map, p_fail, p_dust)
model.gamma = 0.99
model.reward_function = vwreward.VWLinearReward(map)
## Define player
# policy = mdp.agent.HumanAgent(model)
# policy = mdp.agent.RandomAgent(model.A())
opt_policy = mdp.solvers.ValueIteration(100).solve(model)
# policy = mdp.solvers.PolicyIterator(20, mdp.solvers.ExactPolicyEvaluator()).solve(model)
# policy = mdp.solvers.PolicyIterator(20, mdp.solvers.IteratingPolicyEvaluator(100)).solve(model)
# policy = mdp.solvers.PolicyIterator(20, mdp.solvers.SamplingPolicyEvaluator(100, 50)).solve(model)
policy = mdp.solvers.LSPI(50, 5000, vwetc.VWFeatureFunction()).solve(model)
# policy = mdp.solvers.LSPI(50, 5000).solve(model)
## Print
print model.info()
n_different = 0
for s in model.S():
if opt_policy.actions(s) != policy.actions(s):
n_different += 1
print 'Optimal Policy and Approx Policy differ on {} states of {}'.format(n_different, len(model.S()))
## Simulate
print 'Sample run:'
for (s,a,r) in mdp.simulation.simulate(model, policy, initial, t_max):
print '%s, %s, %f' % (s,a,r)