def test_soft_opc_scorer(observation_shape, action_size, n_episodes, episode_length, threshold): # projection matrix for deterministic action A = np.random.random(observation_shape + (action_size, )) episodes = [] for _ in range(n_episodes): observations = np.random.random((episode_length, ) + observation_shape) actions = np.matmul(observations, A).astype('f4') rewards = np.random.random((episode_length, 1)).astype('f4') episode = Episode(observation_shape, action_size, observations.astype('f4'), actions, rewards) episodes.append(episode) algo = DummyAlgo(A, 0.0) success_values = [] all_values = [] for episode in episodes: is_success = episode.compute_return() >= threshold batch = TransitionMiniBatch(episode.transitions) values = algo.predict_value(batch.observations, batch.actions) if is_success: success_values += values.tolist() all_values += values.tolist() scorer = soft_opc_scorer(threshold) score = scorer(algo, episodes) assert np.allclose(score, np.mean(success_values) - np.mean(all_values))
def _get_scorers(discrete_action, dataset_stats): scorers = {} scorers["td_error"] = td_error_scorer scorers["discounted_sum_of_advantage"] = discounted_sum_of_advantage_scorer scorers["value_scale"] = average_value_estimation_scorer scorers["value_standard_deviation"] = value_estimation_std_scorer scorers["initial_state_value"] = initial_state_value_estimation_scorer scorers["soft_opc"] = soft_opc_scorer(0.8 * dataset_stats["return"]["max"]) if discrete_action: scorers["action_match"] = discrete_action_match_scorer else: scorers["action_difference"] = continuous_action_diff_scorer return scorers
from d3rlpy.metrics.scorer import evaluate_on_environment from d3rlpy.metrics.scorer import initial_state_value_estimation_scorer from d3rlpy.metrics.scorer import soft_opc_scorer dataset, env = get_pybullet('hopper-bullet-mixed-v0') train_episodes, test_episodes = train_test_split(dataset, test_size=0.2) # train algorithm cql = CQL(n_epochs=100, use_gpu=True) cql.fit(train_episodes, eval_episodes=test_episodes, scorers={ 'environment': evaluate_on_environment(env), 'init_value': initial_state_value_estimation_scorer, 'soft_opc': soft_opc_scorer(600) }) # or load the trained model # cql = CQL.from_json('<path-to-json>/params.json') # cql.load_model('<path-to-model>/model.pt') # evaluate the trained policy fqe = FQE(algo=cql, n_epochs=200, q_func_factory='qr', learning_rate=1e-4, use_gpu=True, encoder_params={'hidden_units': [1024, 1024, 1024, 1024]}) fqe.fit(dataset.episodes, eval_episodes=dataset.episodes,
dataset, env = get_atari('breakout-expert-v0') train_episodes, test_episodes = train_test_split(dataset, test_size=0.2) # train algorithm cql = DiscreteCQL(n_epochs=100, scaler='pixel', q_func_factory='qr', n_frames=4, use_gpu=True) cql.fit(train_episodes, eval_episodes=test_episodes, scorers={ 'environment': evaluate_on_environment(env, epsilon=0.05), 'init_value': initial_state_value_estimation_scorer, 'soft_opc': soft_opc_scorer(70) }) # or load the trained model # cql = DiscreteCQL.from_json('<path-to-json>/params.json') # cql.load_model('<path-to-model>/model.pt') # evaluate the trained policy fqe = DiscreteFQE(algo=cql, n_epochs=100, q_func_factory='qr', learning_rate=1e-4, scaler='pixel', n_frames=4, discrete_action=True, use_gpu=True)