def test_mujoco_controllers_absolute_step(): print("starting evaluation for {} episodes".format(args.num_eval_episodes)) # generate random seed eval_base_path = os.path.join( results_dir, '_absstep%s_tc%s' % (args.relative_step_size, args.eval_filename_modifier)) eval_step_file_path = eval_base_path + '.cpkl' eval_env = suite.load(domain_name=domain, task_name=task, task_kwargs=task_kwargs, environment_kwargs=environment_kwargs) print("CREATING REPLAY eval", cam_dim) eval_replay_buffer = ReplayBuffer(kwargs['state_dim'], kwargs['action_dim'], max_size=int(args.eval_replay_size), cam_dim=cam_dim, seed=seed) error_dict = {} for jt in range(0, 13): error_dict[jt] = 0 done = False num_steps = 0 reward = 0 state_names_dict = get_state_names_dict() state_type, reward, discount, state = eval_env.reset() frame_compressed = get_next_frame(eval_env) obs_angles = deepcopy(state['observations'][3:13 + 3]) total_errors = 0 direction = 1 obs_angles = deepcopy(state['observations'][3:13 + 3]) action = deepcopy(obs_angles) num_steps = 0 done = False while not done: if obs_angles[jt] - (2 * args.relative_step_size) <= amins[jt]: direction = 1 print("direction", direction, action[jt], amins[jt], amaxes[jt]) if obs_angles[jt] + (2 * args.relative_step_size) >= amaxes[jt]: direction = -1 print("direction", direction, action[jt], amins[jt], amaxes[jt]) action[jt] = obs_angles[jt] + args.relative_step_size * direction print('JT{}N{}A'.format(jt, num_steps), action) reward = 0 # Perform action step_type, _, discount, next_state = eval_env.step(action) last_obs_angles = deepcopy(state['observations'][3:13 + 3]) obs_angles = deepcopy(next_state['observations'][3:13 + 3]) print('JT{}N{}O'.format(jt, num_steps), obs_angles) error = action - obs_angles abs_error = np.abs(error) if np.max(abs_error) > .1: print("----ERROR", error) error_dict[jt] += 1 reward = -1 next_frame_compressed = get_next_frame(eval_env) done = step_type.last() # Store data in replay buffer eval_replay_buffer.add(state['observations'], action, reward, next_state['observations'], done, frame_compressed=frame_compressed, next_frame_compressed=next_frame_compressed) frame_compressed = next_frame_compressed state = next_state num_steps += 1 print("JT TOTAL ERRORS", error_dict) last_steps = eval_replay_buffer.get_last_steps(num_steps) done = True emovie_path = eval_base_path + '_JT%02d_%s_%s.mp4' % ( jt, args.eval_filename_modifier, args.camera_view) ebase = emovie_path.replace('.mp4', '') plotting.plot_states(last_steps, ebase, detail_dict=state_names_dict) plotting.plot_position_actions(last_steps, ebase, relative=args.relative_step) plotting.plot_frames(emovie_path, eval_replay_buffer.get_last_steps(num_steps), plot_action_frames=True, min_action=-kwargs['max_action'], max_action=kwargs['max_action'], plot_frames=True) pickle.dump(eval_replay_buffer, open(ebase + '.epkl', 'wb')) plotting.plot_replay_reward(eval_replay_buffer, ebase, start_step=0, name_modifier='train') # write data files print("---------------------------------------") eval_replay_buffer.shrink_to_last_step() pickle.dump(eval_replay_buffer, open(eval_step_file_path, 'wb')) print("JT TOTAL ERRORS", error_dict) movie_path = eval_base_path + '_%s_%s.mp4' % (args.eval_filename_modifier, args.camera_view) plotting.plot_frames(movie_path, eval_replay_buffer.get_last_steps( eval_replay_buffer.size), plot_action_frames=True, min_action=-kwargs['max_action'], max_action=kwargs['max_action'], plot_frames=False)
def evaluate(load_model_filepath): print("starting evaluation for {} episodes".format(args.num_eval_episodes)) policy, train_step, results_dir, loaded_modelpath = load_policy(load_model_filepath) eval_seed = args.seed+train_step task_kwargs['random'] = eval_seed load_model_base = loaded_modelpath.replace('.pt', '') plotting.plot_loss_dict(policy, load_model_base) state_names_dict = get_state_names_dict() train_replay_buffer = load_replay_buffer(load_model_base + '.pkl') eval_env = suite.load(domain_name=args.domain, task_name=args.task, task_kwargs=task_kwargs, environment_kwargs=environment_kwargs) # generate random seed random_state = np.random.RandomState(eval_seed) train_dir = os.path.join(load_model_base + '_train%s'%args.eval_filename_modifier) if not os.path.exists(train_dir): os.makedirs(train_dir) train_base = os.path.join(train_dir, get_step_filename(train_step)+'_train') plotting.plot_replay_reward(train_replay_buffer, train_base, start_step=train_step, name_modifier='train') plotting.plot_states(train_replay_buffer.get_last_steps(train_replay_buffer.size), train_base, detail_dict=state_names_dict) eval_dir = os.path.join(load_model_base + '_eval%s'%args.eval_filename_modifier) if not os.path.exists(eval_dir): os.makedirs(eval_dir) print('saving results to dir: {}'.format(eval_dir)) eval_base = os.path.join(eval_dir, get_step_filename(train_step)+'_eval_S{:05d}'.format(eval_seed)) eval_step_filepath = eval_base + '%s.epkl'%args.eval_filename_modifier if os.path.exists(eval_step_filepath) and not args.overwrite_replay: print('loading existing replay buffer:{}'.format(eval_step_filepath)) eval_replay_buffer = load_replay_buffer(eval_step_filepath) else: eval_replay_buffer = ReplayBuffer(kwargs['state_dim'], kwargs['action_dim'], max_size=int(args.eval_replay_size), cam_dim=cam_dim, seed=eval_seed) for e in range(args.num_eval_episodes): done = False num_steps = 0 state_type, reward, discount, state = eval_env.reset() frame_compressed = get_next_frame(eval_env) # TODO off by one error in step count!? of replay_buffer while done == False: action = ( policy.select_action(state['observations']) ).clip(-kwargs['max_action'], kwargs['max_action']) # Perform action step_type, reward, discount, next_state = eval_env.step(action) next_frame_compressed = get_next_frame(eval_env) done = step_type.last() # Store data in replay buffer eval_replay_buffer.add(state['observations'], action, reward, next_state['observations'], done, frame_compressed=frame_compressed, next_frame_compressed=next_frame_compressed) frame_compressed = next_frame_compressed state = next_state num_steps+=1 time.sleep(.1) # plot episode er = np.int(eval_replay_buffer.episode_rewards[-1]) epath = eval_base+ '_E{}_R{}'.format(e, er) exp = eval_replay_buffer.get_last_steps(num_steps) plotting.plot_states(exp, epath, detail_dict=state_names_dict) if args.domain == 'jaco': plotting.plot_position_actions(exp, epath, relative=True) if np.max([args.plot_movie, args.plot_action_movie, args.plot_frames]): emovie_path = epath+'CAM{}.mp4'.format(e, er, args.camera_view) print('plotting episode: {}'.format(emovie_path)) plotting.plot_frames(emovie_path, eval_replay_buffer.get_last_steps(num_steps), plot_action_frames=args.plot_action_movie, min_action=-kwargs['max_action'], max_action=kwargs['max_action'], plot_frames=args.plot_frames) eval_replay_buffer.shrink_to_last_step() pickle.dump(eval_replay_buffer, open(eval_step_filepath, 'wb')) # plot evaluation plotting.plot_replay_reward(eval_replay_buffer, eval_base, start_step=train_step, name_modifier='eval') plotting.plot_states(eval_replay_buffer.get_last_steps(eval_replay_buffer.size), eval_base, detail_dict=state_names_dict) if np.max([args.plot_movie, args.plot_action_movie, args.plot_frames]): movie_path = eval_base+'_CAM{}.mp4'.format(args.camera_view) plotting.plot_frames(movie_path, eval_replay_buffer.get_last_steps(eval_replay_buffer.size), plot_action_frames=args.plot_action_movie, min_action=-kwargs['max_action'], max_action=kwargs['max_action'], plot_frames=args.plot_frames) return eval_replay_buffer, eval_step_filepath
def test_mujoco_controllers_relative_step(): print("starting evaluation for {} episodes".format(args.num_eval_episodes)) # generate random seed eval_base_path = os.path.join( results_dir, '_relstep%s_tc%s' % (args.relative_step, args.eval_filename_modifier)) eval_step_file_path = eval_base_path + '.cpkl' eval_env = suite.load(domain_name=domain, task_name=task, task_kwargs=task_kwargs, environment_kwargs=environment_kwargs) print("CREATING REPLAY eval", cam_dim) eval_replay_buffer = ReplayBuffer(kwargs['state_dim'], kwargs['action_dim'], max_size=int(args.eval_replay_size), cam_dim=cam_dim, seed=seed) print(amins) print(amaxes) error_dict = {} for jt in range(0, 13): error_dict[jt] = 0 done = False num_steps = 0 reward = 0 state_names_dict = get_state_names_dict() state_type, reward, discount, state = eval_env.reset() frame_compressed = get_next_frame(eval_env) obs_angles = deepcopy(state['observations'][3:13 + 3]) total_errors = 0 # WHAT I KNOW # if I dont step, nothing changes # when I run named.data.qpos[home] - the robot looks like i want # however if from that home, I "step" to "home" (the same angles) - the result is 45 offset """ dm_control's rl/control.py var CONTROL_TIMESTEP is not ideal for position controllers. We need a timestep of .001 (specified in the xml file) for the physics solver in order to get it to work. At each timestep, the robot has CONTROL_TIMESTEP seconds to reach a particular position. We interface with the real robot by calling an action client which blocks until the position is reached. I think the best way to make these to processes appear the same is to carefully choose CONTROL_TIMESTEP and only allow sequential actions to be relatively near to each other so as to ensure that they can be completed within CONTROL_TIMESTEP seconds in mujoco. The two systems will have different types of errors if mujoco is misconfigured. The pararms that effect performance for position controllers (as far as I can tell) are: CONTROL_TIMESTEP sent to rl/control.py (time allowed to reach goal) kv gain in the xml file physics timestep in xml file and how far the desired action is from the current position Here are some performance stats with those variables tested: +1 error count if an observed position angle > .1 of the commanded angle """ direction = 1 action = np.zeros(13) obs_angles = deepcopy(state['observations'][3:13 + 3]) num_steps = 0 done = False while not done: if obs_angles[jt] - (2 * args.relative_step_size) <= amins[jt]: direction = 1 print("direction", direction, action[jt], amins[jt], amaxes[jt]) if obs_angles[jt] + (2 * args.relative_step_size) >= amaxes[jt]: direction = -1 print("direction", direction, action[jt], amins[jt], amaxes[jt]) # turn hand so it is easier to see for finger moves #if num_steps < 10: # action[1] = .1 # action[3] = .01 #if jt == 4: # if num_steps < 10: # action[3] = .1 # else: # action[3] = 0.0 #if jt > 6: # if num_steps < 10: # action[5] = -.1 # else: # action[5] = 0.0 action[jt] = args.relative_step_size * direction #base_action[:7] = action print('JT{}N{}A'.format(jt, num_steps), action) reward = 0 # Perform action step_type, _, discount, next_state = eval_env.step(action) last_obs_angles = deepcopy(state['observations'][3:13 + 3]) obs_angles = deepcopy(next_state['observations'][3:13 + 3]) print('JT{}N{}O'.format(jt, num_steps), obs_angles) error = (last_obs_angles + action) - obs_angles abs_error = np.abs(error) if np.max(abs_error) > .1: print("----ERROR", error) error_dict[jt] += 1 reward = -1 next_frame_compressed = get_next_frame(eval_env) done = step_type.last() # Store data in replay buffer eval_replay_buffer.add(state['observations'], action, reward, next_state['observations'], done, frame_compressed=frame_compressed, next_frame_compressed=next_frame_compressed) frame_compressed = next_frame_compressed state = next_state num_steps += 1 print("JT TOTAL ERRORS", error_dict) last_steps = eval_replay_buffer.get_last_steps(num_steps) done = True emovie_path = eval_base_path + '_JT%02d_%s_%s.mp4' % ( jt, args.eval_filename_modifier, args.camera_view) ebase = emovie_path.replace('.mp4', '') plotting.plot_states(last_steps, ebase, detail_dict=state_names_dict) plotting.plot_position_actions(last_steps, ebase, relative=args.relative_step) plotting.plot_frames(emovie_path, eval_replay_buffer.get_last_steps(num_steps), plot_action_frames=True, min_action=-kwargs['max_action'], max_action=kwargs['max_action'], plot_frames=True) pickle.dump(eval_replay_buffer, open(ebase + '.epkl', 'wb')) plotting.plot_replay_reward(eval_replay_buffer, ebase, start_step=0, name_modifier='train') # write data files print("---------------------------------------") eval_replay_buffer.shrink_to_last_step() pickle.dump(eval_replay_buffer, open(eval_step_file_path, 'wb')) print("JT TOTAL ERRORS", error_dict) movie_path = eval_base_path + '_%s_%s.mp4' % (args.eval_filename_modifier, args.camera_view) plotting.plot_frames(movie_path, eval_replay_buffer.get_last_steps( eval_replay_buffer.size), plot_action_frames=True, min_action=-kwargs['max_action'], max_action=kwargs['max_action'], plot_frames=False)