def backward_induction_vf_and_pi( self, ff: Sequence[Callable[[float], float]] ) -> Iterator[Tuple[DNNApprox[float], Policy[float, float]]]: init_fa: DNNApprox[float] = self.get_vf_func_approx(ff) mdp_f0_mu_triples: Sequence[ Tuple[ MarkovDecisionProcess[float, float], DNNApprox[float], SampledDistribution[float], ] ] = [ (self.get_mdp(i), init_fa, self.get_states_distribution(i)) for i in range(self.time_steps()) ] num_state_samples: int = 300 error_tolerance: float = 1e-8 return back_opt_vf_and_policy( mdp_f0_mu_triples=mdp_f0_mu_triples, γ=1.0, num_state_samples=num_state_samples, error_tolerance=error_tolerance, )
def backward_induction_vf_and_pi( self, features: Sequence[Callable[[StateType], float]], reg_coeff: float ) -> Iterator[Tuple[FunctionApprox[StateType], Policy[StateType, bool]]]: mdp_f0_mu_triples: Sequence[Tuple[MarkovDecisionProcess[StateType, bool], FunctionApprox[StateType], SampledDistribution[StateType]]] = [ (self.get_mdp(t=i), self.get_vf_func_approx( t=i, features=features, reg_coeff=reg_coeff), self.get_states_distribution( t=i)) for i in range(self.num_steps + 1) ] num_state_samples: int = 1000 return back_opt_vf_and_policy(mdp_f0_mu_triples=mdp_f0_mu_triples, γ=np.exp(-self.rate * self.expiry / self.num_steps), num_state_samples=num_state_samples, error_tolerance=1e-8)
def test_value_iteration(self): vpstar = optimal_vf_and_policy(self.mdp_seq, 1.) states = self.single_step_mdp.states() fa_dynamic = Dynamic({s: 0.0 for s in states}) fa_tabular = Tabular() distribution = Choose(set(states)) approx_vpstar_finite = back_opt_vf_and_policy_finite( [(self.mdp_seq[i], fa_dynamic) for i in range(self.steps)], 1. ) approx_vpstar = back_opt_vf_and_policy( [(self.single_step_mdp, fa_tabular, distribution) for _ in range(self.steps)], 1., num_state_samples=120, error_tolerance=0.01 ) for t, ((v1, _), (v2, _), (v3, _)) in enumerate(zip( vpstar, approx_vpstar_finite, approx_vpstar )): states = self.mdp_seq[t].keys() v1_arr = np.array([v1[s] for s in states]) v2_arr = v2.evaluate(states) v3_arr = v3.evaluate(states) self.assertLess(max(abs(v1_arr - v2_arr)), 0.001) self.assertLess(max(abs(v1_arr - v3_arr)), 1.0)
def backward_induction_vf_and_pi( self ) -> Iterator[Tuple[ValueFunctionApprox[PriceAndShares], DeterministicPolicy[PriceAndShares, int]]]: mdp_f0_mu_triples: Sequence[ Tuple[MarkovDecisionProcess[PriceAndShares, int], ValueFunctionApprox[PriceAndShares], SampledDistribution[NonTerminal[PriceAndShares]]]] = [ (self.get_mdp(i), self.func_approx, self.get_states_distribution(i)) for i in range(self.time_steps) ] num_state_samples: int = 10000 error_tolerance: float = 1e-6 return back_opt_vf_and_policy(mdp_f0_mu_triples=mdp_f0_mu_triples, γ=self.discount_factor, num_state_samples=num_state_samples, error_tolerance=error_tolerance)