def __init__(self, args, process_ind=0, num_envs_per_process=1):
super(AtariEnv, self).__init__(args, process_ind, num_envs_per_process)
# env_params for this env
assert self.num_envs_per_process == 1
self.seed = self.seed + self.process_ind * self.num_envs_per_actor # NOTE: check again
# setup ale
self.ale = atari_py.ALEInterface()
self.ale.setInt('random_seed', self.seed)
self.ale.setInt('max_num_frames', self.early_stop)
self.ale.setFloat('repeat_action_probability',
0) # Disable sticky actions
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
print(atari_py.get_game_path(self.game))
self.ale.loadROM(atari_py.get_game_path(
self.game)) # ROM loading must be done after setting options
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e]
for i, e in zip(range(len(actions)), actions))
self.lives = 0 # life counter (used in DeepMind training)
self.just_died = False # when lost one life, but game is still not over
# setup
self.exp_state1 = deque(maxlen=self.state_cha)
self._reset_experience()
def __init__(
self,
game,
seed,
device,
training=True,
clip_rewards_val=1,
history_length=4,
sticky_action_p=0,
max_episode_length=108e3,
):
# pylint: enable=bad-continuation
self.game_name = game
self.device = device
self.ale = atari_py.ALEInterface()
self.ale.setInt("random_seed", seed)
self.ale.setInt("max_num_frames_per_episode", max_episode_length)
self.ale.setFloat("repeat_action_probability",
sticky_action_p) # Disable sticky actions
self.ale.setInt("frame_skip", 0)
self.ale.setBool("color_averaging", False)
self.ale.loadROM(atari_py.get_game_path(
game)) # ROM loading must be done after setting options
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e]
for i, e in zip(range(len(actions)), actions))
self.action_space = Discrete(len(self.actions))
self.lives = 0 # Life counter (used in DeepMind training)
# Used to check if resetting only from loss of life
self.life_termination = False
self.window = history_length # Number of frames to concatenate
self.state_buffer = deque([], maxlen=history_length)
self.training = training # Consistent with model training mode
self.clip_val = clip_rewards_val
self.sticky_action_p = sticky_action_p
def __init__(self, game='pong', obs_type='ram', frameskip=(2, 5), repeat_action_probability=0.):
"""Frameskip should be either a tuple (indicating a random range to
choose from, with the top value exclude), or an int."""
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image')
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
raise IOError('You asked for game %s but path %s does not exist'%(game, self.game_path))
self._obs_type = obs_type
self.frameskip = frameskip
self.ale = atari_py.ALEInterface()
self.viewer = None
# Tune (or disable) ALE's action repeat:
# https://github.com/openai/gym/issues/349
assert isinstance(repeat_action_probability, (float, int)), "Invalid repeat_action_probability: {!r}".format(repeat_action_probability)
self.ale.setFloat('repeat_action_probability'.encode('utf-8'), repeat_action_probability)
self._seed()
(screen_width, screen_height) = self.ale.getScreenDims()
self._buffer = np.empty((screen_height, screen_width, 4), dtype=np.uint8)
self._action_set = self.ale.getMinimalActionSet()
self.action_space = spaces.Discrete(len(self._action_set))
(screen_width,screen_height) = self.ale.getScreenDims()
if self._obs_type == 'ram':
self.observation_space = spaces.Box(low=np.zeros(128), high=np.zeros(128)+255)
elif self._obs_type == 'image':
self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 3))
else:
raise error.Error('Unrecognized observation type: {}'.format(self._obs_type))
def __init__(self, args):
self.device = args.device
self.ale = atari_py.ALEInterface()
self.ale.setInt('random_seed', args.seed)
self.ale.setInt('max_num_frames_per_episode', args.max_episode_length)
self.ale.setFloat('repeat_action_probability', 0)
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
self.ale.loadROM(atari_py.get_game_path(args.game))
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e] for i, e in zip(range(len(actions)), actions))
self.lives = 0
self.life_termination = False
self.window = args.history_length
self.state_buffer = deque([], maxlen=args.history_length)
self.training = True
self.encode_trans = args.encode_transitions
self.useVLAE = args.use_encoder
self.dataset = AtariDataset(transition=self.encode_trans)
self.network = VLadder(self.dataset, file_path='vlae/models/', name=args.name, add_coords=True)
if self.useVLAE == 0 or self.useVLAE == 2:
self.xdim = 84
self.ydim = 84
elif self.useVLAE == 1:
self.xdim = 1
self.ydim = 84
def __init__(self, mask_num=0 ,game='pong', obs_type='ram', frameskip=(2, 5), repeat_action_probability=0.):
"""Frameskip should be either a tuple (indicating a random range to
choose from, with the top value exclude), or an int."""
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image')
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
raise IOError('You asked for game %s but path %s does not exist'%(game, self.game_path))
self._obs_type = obs_type
self.frameskip = frameskip
self.ale = atari_py.ALEInterface()
self.viewer = None
# Tune (or disable) ALE's action repeat:
# https://github.com/openai/gym/issues/349
assert isinstance(repeat_action_probability, (float, int)), "Invalid repeat_action_probability: {!r}".format(repeat_action_probability)
self.ale.setFloat('repeat_action_probability'.encode('utf-8'), repeat_action_probability)
self._seed()
(screen_width, screen_height) = self.ale.getScreenDims()
self._action_set = self.ale.getMinimalActionSet()
self.mask_num = mask_num
def __init__(self, args, training=True):
self.device = args.device
self.ale = atari_py.ALEInterface()
seed = args.seed if training else args.seed + 1 # different seeds for training and evaluation
self.ale.setInt('random_seed', seed)
if training:
self.ale.setInt('max_num_frames_per_episode',
args.max_episode_length)
else:
self.ale.setInt('max_num_frames_per_episode', 108e3)
self.ale.setFloat('repeat_action_probability',
0) # Disable sticky actions
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
self.ale.loadROM(atari_py.get_game_path(
args.game)) # ROM loading must be done after setting options
self.game = args.game
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e]
for i, e in zip(range(len(actions)), actions))
self.lives = 0 # Life counter (used in DeepMind training)
self.life_termination = False # Used to check if resetting only from loss of life
self.window = args.history_length # Number of frames to concatenate
self.state_buffer = deque([], maxlen=args.history_length)
self.training = training # Consistent with model training mode
def __init__(self, args):
self.device = args.device
self.ale = atari_py.ALEInterface()
self.ale.setInt("random_seed", args.seed)
self.ale.setInt("max_num_frames_per_episode", args.max_episode_length)
self.ale.setFloat("repeat_action_probability",
args.proba_sticky_actions)
self.ale.setInt("frame_skip", 0)
self.ale.setBool("color_averaging", False)
# ROM loading must be done after setting options
self.ale.loadROM(atari_py.get_game_path(args.game))
actions = self.ale.getLegalActionSet(
) # We always use 18 actions. See revisiting ALE.
self.actions = {i: e for i, e in zip(range(len(actions)), actions)}
self.window = args.history_length # Number of frames to concatenate
self.state_buffer = deque([], maxlen=args.history_length)
self.action_repeat = args.action_repeat
self.SABER_mode = not args.disable_SABER_mode
if self.SABER_mode:
# We need to divide time by action repeat to get the max_step_stuck
self.max_step_stuck_SABER = int(args.max_frame_stuck_SABER /
self.action_repeat)
# Reward on defender are really weird at the time we write this code (7 August 2019).
# All rewards are basically multiplied by 100 and there is always a initial
# reward of 10 both for no reason
if args.game == "defender":
self.handle_bug_in_defender = True
else:
self.handle_bug_in_defender = False
def __init__(self,
game_name,
seed,
max_episode_length=1e10,
history_length=4,
reward_clip=1,
device='cpu'):
self.device = device
self.ale = atari_py.ALEInterface()
self.ale.setInt('random_seed', seed)
self.ale.setInt('max_num_frames_per_episode', max_episode_length)
self.ale.setFloat('repeat_action_probability',
0) # Disable sticky actions
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
self.ale.loadROM(atari_py.get_game_path(
game_name)) # ROM loading must be done after setting options
actions = self.ale.getMinimalActionSet()
self.actions = dict(zip(range(len(actions)), actions))
self.reward_clip = reward_clip
self.lives = 0 # Life counter (used in DeepMind training)
self.life_termination = False # Used to check if resetting only from loss of life
self.window = history_length # Number of frames to concatenate
self.state_buffer = deque([], maxlen=history_length)
self.training = True # Consistent with model training mode
self.viewer = None
def __init__(self, game='pong', obs_type='ram', frameskip=(2, 5), repeat_action_probability=0.):
"""Frameskip should be either a tuple (indicating a random range to
choose from, with the top value exclude), or an int."""
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image')
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
raise IOError('You asked for game %s but path %s does not exist'%(game, self.game_path))
self._obs_type = obs_type
self.frameskip = frameskip
self.ale = atari_py.ALEInterface()
self.viewer = None
# Tune (or disable) ALE's action repeat:
# https://github.com/openai/gym/issues/349
assert isinstance(repeat_action_probability, (float, int)), "Invalid repeat_action_probability: {!r}".format(repeat_action_probability)
self.ale.setFloat('repeat_action_probability'.encode('utf-8'), repeat_action_probability)
self.seed()
(screen_width, screen_height) = self.ale.getScreenDims()
self._action_set = self.ale.getMinimalActionSet()
self.action_space = spaces.Discrete(len(self._action_set))
(screen_width,screen_height) = self.ale.getScreenDims()
if self._obs_type == 'ram':
self.observation_space = spaces.Box(low=0, high=255, dtype=np.uint8, shape=(128,))
elif self._obs_type == 'image':
self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 3), dtype=np.uint8)
else:
raise error.Error('Unrecognized observation type: {}'.format(self._obs_type))
def get_num_actions(rom_path, rom_name):
#import os
#print os.path.abspath(atari_py.__file__)
game_path = atari_py.get_game_path(rom_name)
ale = atari_py.ALEInterface()
ale.loadROM(game_path)
return ale.getMinimalActionSet()
def __init__(self, args):
self.ale = atari_py.ALEInterface()
self.ale.setInt('random_seed', args.seed)
self.ale.setInt('max_num_frames_per_episode', args.max_episode_length)
self.ale.setFloat('repeat_action_probability',
0) # Disable sticky actions
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
self.ale.loadROM(atari_py.get_game_path(
args.game)) # ROM loading must be done after setting options
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e]
for i, e in zip(range(len(actions)), actions))
self.lives = 0 # Life counter (used in DeepMind training)
self.life_termination = False # Used to check if resetting only from loss of life
self.window = args.framestack # Number of frames to concatenate
self.state_buffer = deque([], maxlen=args.framestack)
self.training = True # Consistent with model training mode
self.grayscale = args.grayscale
channels = 1 if self.grayscale else 3
self.observation_space = gym.spaces.Box(low=0,
high=255,
dtype=np.uint8,
shape=(args.framestack,
channels, 96, 96))
self.action_space = gym.spaces.Discrete(len(self.actions))
def __init__(self,
name,
seed,
device,
max_episode_length,
frame_stack,
dual_states,
noops=True):
self.device = device
self.ale = atari_py.ALEInterface()
self.ale.setInt('random_seed', seed)
self.ale.setInt('max_num_frames_per_episode', 108e3)
self.ale.setFloat('repeat_action_probability',
0) # Disable sticky actions
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
self.ale.loadROM(atari_py.get_game_path(
name)) # ROM loading must be done after setting options
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e]
for i, e in zip(range(len(actions)), actions))
self.lives = 0 # Life counter (used in DeepMind training)
self.life_termination = False # Used to check if resetting only from loss of life
self.window = frame_stack # Number of frames to concatenate
self.state_buffer = deque([], maxlen=frame_stack)
self.training = True # Consistent with model training mode
self.dual_states = dual_states # Whether to return unedited state as well
self.steps = 0
self.max_steps = max_episode_length
self.noops = noops
def __init__(self, game='pong', obs_type='ram'):
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image', 'gray')
game_path = atari_py.get_game_path(game)
if not os.path.exists(game_path):
raise IOError('You asked for game %s but path %s does not exist'%(game, game_path))
self.ale = atari_py.ALEInterface()
self.ale.loadROM(game_path)
self._obs_type = obs_type
self._action_set = self.ale.getMinimalActionSet()
self.viewer = None
(screen_width,screen_height) = self.ale.getScreenDims()
self.action_space = spaces.Discrete(len(self._action_set))
if self._obs_type == 'ram':
self.observation_space = spaces.Box(low=np.zeros(128), high=np.zeros(128)+255)
elif self._obs_type == 'image':
self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 3))
self._screen = np.zeros((screen_height, screen_width, 4), dtype=np.uint8)
elif self._obs_type == 'gray':
self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 1))
self._screen = np.zeros((screen_height, screen_width, 1), dtype=np.uint8)
else:
raise error.Error('Unrecognized observation type: {}'.format(self._obs_type))
def __init__(self, rom_path, rom_name, visualize, actor_id, rseed, single_life_episode = False):
self.ale = atari_py.ALEInterface()
self.ale.setInt("random_seed", rseed * (actor_id +1))
# For fuller control on explicit action repeat (>= ALE 0.5.0)
self.ale.setFloat("repeat_action_probability", 0.0)
# See: http://is.gd/tYzVpj
self.ale.setInt("frame_skip", 4)
#self.ale.setBool("color_averaging", False)
self.ale.loadROM(atari_py.get_game_path(rom_name))
self.legal_actions = self.ale.getMinimalActionSet()
self.single_life_episode = single_life_episode
self.initial_lives = self.ale.lives()
# Processed frames that will be fed in to the network
# (i.e., four 84x84 images)
self.processed_imgs = np.zeros((IMG_SIZE_X, IMG_SIZE_Y,
NR_IMAGES), dtype=np.uint8)
self.screen_width,self.screen_height = self.ale.getScreenDims()
self.rgb_screen = np.zeros((self.screen_height,self.screen_width, 4), dtype=np.uint8)
self.gray_screen = np.zeros((self.screen_height,self.screen_width,1), dtype=np.uint8)
self.visualize = visualize
self.visualize_processed = False
rendering_imported = False
def create_atari_environment(game_name=None, sticky_actions=True, frame_stack=4, screen_size=84, seed=0):
"""Wraps an Atari 2600 Gym environment with some basic preprocessing.
This preprocessing matches the guidelines proposed in Machado et al. (2017),
"Revisiting the Arcade Learning Environment: Evaluation Protocols and Open
Problems for General Agents".
The created environment is the Gym wrapper around the Arcade Learning
Environment.
The main choice available to the user is whether to use sticky actions or not.
Sticky actions, as prescribed by Machado et al., cause actions to persist
with some probability (0.25) when a new command is sent to the ALE. This
can be viewed as introducing a mild form of stochasticity in the environment.
We use them by default.
Args:
game_name: str, the name of the Atari 2600 domain.
sticky_actions: bool, whether to use sticky_actions as per Machado et al.
Returns:
An Atari 2600 environment with some standard preprocessing.
"""
assert game_name is not None
game_path = atari_py.get_game_path(game_name)
if not os.path.exists(game_path):
raise IOError("You asked for game {} but path {} does not "
" exist".format(game_name, game_path))
ale = atari_py.ALEInterface()
ale.setInt(b'random_seed', seed)
repeat_action_prob = 0.25 if sticky_actions else 0
ale.setFloat(b'repeat_action_probability', repeat_action_prob)
ale.loadROM(game_path)
env = AtariPreprocessing(ale, frame_stack=frame_stack, screen_size=screen_size)
return env
def __init__(self, game='pong', obs_type='ram'):
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image')
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
raise IOError('You asked for game %s but path %s does not exist' %
(game, self.game_path))
self._obs_type = obs_type
self.ale = atari_py.ALEInterface()
self.viewer = None
self._seed()
self._action_set = self.ale.getMinimalActionSet()
self.action_space = spaces.Discrete(len(self._action_set))
(screen_width, screen_height) = self.ale.getScreenDims()
if self._obs_type == 'ram':
self.observation_space = spaces.Box(low=np.zeros(128),
high=np.zeros(128) + 255)
elif self._obs_type == 'image':
self.observation_space = spaces.Box(low=0,
high=255,
shape=(screen_height,
screen_width, 3))
else:
raise error.Error('Unrecognized observation type: {}'.format(
self._obs_type))
def __init__(self, game, seed=None, use_sdl=False, n_last_screens=4,
frame_skip=4, treat_life_lost_as_terminal=True,
crop_or_scale='scale', max_start_nullops=30,
record_screen_dir=None):
assert crop_or_scale in ['crop', 'scale']
assert frame_skip >= 1
self.n_last_screens = n_last_screens
self.treat_life_lost_as_terminal = treat_life_lost_as_terminal
self.crop_or_scale = crop_or_scale
self.max_start_nullops = max_start_nullops
# atari_py is used only to provide rom files. atari_py has its own
# ale_python_interface, but it is obsolete.
if not atari_py_available:
raise RuntimeError(
'You need to install atari_py>=0.1.1 to use ALE.')
game_path = atari_py.get_game_path(game)
ale = atari_py.ALEInterface()
if seed is not None:
assert seed >= 0 and seed < 2 ** 31, \
"ALE's random seed must be in [0, 2 ** 31)."
else:
# Use numpy's random state
seed = np.random.randint(0, 2 ** 31)
ale.setInt(b'random_seed', seed)
ale.setFloat(b'repeat_action_probability', 0.0)
ale.setBool(b'color_averaging', False)
if record_screen_dir is not None:
ale.setString(b'record_screen_dir',
str.encode(str(record_screen_dir)))
self.frame_skip = frame_skip
if use_sdl:
if 'DISPLAY' not in os.environ:
raise RuntimeError(
'Please set DISPLAY environment variable for use_sdl=True')
# SDL settings below are from the ALE python example
if sys.platform == 'darwin':
import pygame
pygame.init()
ale.setBool(b'sound', False) # Sound doesn't work on OSX
elif sys.platform.startswith('linux'):
ale.setBool(b'sound', True)
ale.setBool(b'display_screen', True)
ale.loadROM(str.encode(str(game_path)))
assert ale.getFrameNumber() == 0
self.ale = ale
self.legal_actions = ale.getMinimalActionSet()
self.initialize()
self.action_space = spaces.Discrete(len(self.legal_actions))
one_screen_observation_space = spaces.Box(
low=0, high=255,
shape=(84, 84), dtype=np.uint8,
)
self.observation_space = spaces.Tuple(
[one_screen_observation_space] * n_last_screens)
def __init__(self, game='pong', obs_type='ram', frameskip=(2, 5)):
"""Frameskip should be either a tuple (indicating a random range to
choose from, with the top value exclude), or an int."""
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image')
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
raise IOError('You asked for game %s but path %s does not exist'%(game, self.game_path))
self._obs_type = obs_type
self.frameskip = frameskip
self.ale = atari_py.ALEInterface()
self.viewer = None
self._seed()
(screen_width, screen_height) = self.ale.getScreenDims()
self._buffer = np.empty((screen_height, screen_width, 4), dtype=np.uint8)
self._action_set = self.ale.getMinimalActionSet()
self.action_space = spaces.Discrete(len(self._action_set))
(screen_width,screen_height) = self.ale.getScreenDims()
if self._obs_type == 'ram':
self.observation_space = spaces.Box(low=np.zeros(128), high=np.zeros(128)+255)
elif self._obs_type == 'image':
self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 3))
else:
raise error.Error('Unrecognized observation type: {}'.format(self._obs_type))
def __init__(self,
game='pong',
mode=None,
difficulty=None,
obs_type='image',
stack_size=4,
frameskip=(2, 5),
repeat_action_probability=0.,
full_action_space=True,
server_num=0,
img_dim=64,
**kwargs):
"""Frameskip should be either a tuple (indicating a random range to
choose from, with the top value exclude), or an int."""
#utils.EzPickle.__init__( self, game, mode, difficulty, obs_type,
# frameskip, repeat_action_probability)
assert obs_type in ('ram', 'image')
self.game = game
self.game_path = atari_py.get_game_path(game)
self.game_mode = mode
self.game_difficulty = difficulty
self.server_num = server_num
self.img_dim = img_dim
self.stack_size = stack_size
self.episode = 0
if not os.path.exists(self.game_path):
msg = 'You asked for game %s but path %s does not exist'
raise IOError(msg % (game, self.game_path))
self._obs_type = obs_type
self.frameskip = frameskip
self.ale = atari_py.ALEInterface()
self.viewer = None
# Tune (or disable) ALE's action repeat:
# https://github.com/openai/gym/issues/349
assert isinstance(repeat_action_probability, (float, int)), \
"Invalid repeat_action_probability: {!r}".format(repeat_action_probability)
self.ale.setFloat('repeat_action_probability'.encode('utf-8'),
repeat_action_probability)
self.seed()
self._action_set = (self.ale.getLegalActionSet() if full_action_space
else self.ale.getMinimalActionSet())
#self.action_space = spaces.Discrete(len(self._action_set))
(screen_width, screen_height) = self.ale.getScreenDims()
if self._obs_type == 'ram':
#self.observation_space = spaces.Box(low=0, high=255, dtype=np.uint8, shape=(128,))
pass
elif self._obs_type == 'image':
#self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 3), dtype=np.uint8)
pass
else:
raise ValueError('Unrecognized observation type: {}'.format(
self._obs_type))
def __init__(self,
game="pong",
frame_skip=4, # Frames per step (>=1).
num_img_obs=4, # Number of (past) frames in observation (>=1) - "frame stacking".
clip_reward=True,
episodic_lives=True,
fire_on_reset=False,
max_start_noops=30,
repeat_action_probability=0.,
horizon=27000,
no_extrinsic=False,
no_negative_reward=False,
normalize_obs=False,
normalize_obs_steps=10000,
downsampling_scheme='classical',
record_freq=0,
record_dir=None
):
save__init__args(locals(), underscore=True)
# ALE
game_path = atari_py.get_game_path(game)
if not os.path.exists(game_path):
raise IOError("You asked for game {} but path {} does not "
" exist".format(game, game_path))
self.ale = atari_py.ALEInterface()
self.ale.setFloat(b'repeat_action_probability', repeat_action_probability)
self.ale.loadROM(game_path)
# Spaces
self._action_set = self.ale.getMinimalActionSet()
self._action_space = IntBox(low=0, high=len(self._action_set))
if downsampling_scheme == 'classical':
self._frame_shape = (84, 84) # (W, H)
elif downsampling_scheme == 'new':
self._frame_shape = (80, 104)
obs_shape = (num_img_obs, self._frame_shape[1], self._frame_shape[0])
self._observation_space = IntBox(low=0, high=255, shape=obs_shape, dtype="uint8")
self._max_frame = self.ale.getScreenGrayscale()
self._raw_frame_1 = self._max_frame.copy()
self._raw_frame_2 = self._max_frame.copy()
self._obs = np.zeros(shape=obs_shape, dtype="uint8")
# Settings
self._has_fire = "FIRE" in self.get_action_meanings()
self._has_up = "UP" in self.get_action_meanings()
self._horizon = int(horizon)
# Recording
self.record_env = False # set in samping_process for environment 0
self._record_episode = False
self._record_freq = record_freq
self._video_dir = os.path.join(record_dir, 'videos')
self._frames_dir = os.path.join(self._video_dir, 'frames')
self._episode_number = 0
self.reset()
def __init__(self):
self.ale = ALEInterface();
self.ale.loadROM(get_game_path('boxing'));
self.legal_actions = self.ale.getMinimalActionSet();
self.policyModel = PolicyModel(self.legal_actions);
#load model
if True == os.path.exists('model'): self.policyModel.load_weights('./model/vpg_model');
self.status_size_ = 4
self.gamma_ = 1; #the reward it too small
def __init__(self, **kwargs):
game = kwargs['game']
frame_skip = kwargs['frame_skip']
max_start_nullops = kwargs['max_start_nullops']
death_ends_episode = kwargs['death_ends_episode']
hist_len = kwargs['hist_len']
crop_top = kwargs['crop_top']
crop_bottom = kwargs['crop_bottom']
seed = kwargs.get('seed', 0)
rng = kwargs.get('rng', None)
self.train_epoch_length = kwargs['train_epoch_length']
self.test_epoch_length = kwargs['test_epoch_length']
self.epoch_length = None
self.epoch_steps = 0
self.unwrapped = self
self.ale = ale_python_interface.ALEInterface()
# Implement frame skip ourselves
self.frame_skip = frame_skip
self.ale.setInt("frame_skip", 1)
self.ale.setBool('display_screen', False)
self.ale.setFloat('repeat_action_probability', 0.)
self.ale.loadROM(atari_py.get_game_path(game))
self.max_start_nullops = max_start_nullops
self.actions = self.ale.getMinimalActionSet()
# Fire button not required for deterministic pong
if game == 'pong':
self.actions = [0, 3, 4]
self.observation_space = gym.spaces.Box(low=0,
high=255,
shape=(84, 84, hist_len))
self.action_space = gym.spaces.Discrete(len(self.actions))
self.width, self.height = self.ale.getScreenDims()
self.buffer_length = 2
self.buffer_count = 0
self.screen_buffer = np.empty(
(self.buffer_length, self.height, self.width), dtype=np.uint8)
self.observations = np.zeros((84, 84, hist_len), dtype=np.float32)
self.death_ends_episode = death_ends_episode
self.terminal_lol = False # Most recent episode ended on a loss of life
self.crop_top = crop_top
self.crop_bottom = crop_bottom
if rng is None:
self.rng = np.random.RandomState(seed)
else:
self.rng = rng
def __init__(self, game='pong', obs_type='ram'):
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image')
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
raise IOError('You asked for game %s but path %s does not exist'%(game, self.game_path))
self._obs_type = obs_type
self.ale = atari_py.ALEInterface()
self.viewer = None
self._seed()
def __init__(
self,
game="pong",
frame_skip=4, # Frames per step (>=1).
num_img_obs=4, # Number of (past) frames in observation (>=1).
clip_reward=True,
episodic_lives=True,
max_start_noops=30,
repeat_action_probability=0.,
horizon=27000,
stack_actions=0,
grayscale=True,
imagesize=84,
seed=42,
id=0,
):
save__init__args(locals(), underscore=True)
# ALE
game_path = atari_py.get_game_path(game)
if not os.path.exists(game_path):
raise IOError("You asked for game {} but path {} does not "
" exist".format(game, game_path))
self.ale = atari_py.ALEInterface()
self.seed(seed, id)
self.ale.setFloat(b'repeat_action_probability',
repeat_action_probability)
self.ale.loadROM(game_path)
# Spaces
self.stack_actions = stack_actions
self._action_set = self.ale.getMinimalActionSet()
self._action_space = IntBox(low=0, high=len(self._action_set))
self.channels = 1 if grayscale else 3
self.grayscale = grayscale
self.imagesize = imagesize
if self.stack_actions: self.channels += 1
obs_shape = (num_img_obs, self.channels, imagesize, imagesize)
self._observation_space = IntBox(low=0,
high=255,
shape=obs_shape,
dtype="uint8")
self._max_frame = self.ale.getScreenGrayscale() if self.grayscale \
else self.ale.getScreenRGB()
self._raw_frame_1 = self._max_frame.copy()
self._raw_frame_2 = self._max_frame.copy()
self._obs = np.zeros(shape=obs_shape, dtype="uint8")
# Settings
self._has_fire = "FIRE" in self.get_action_meanings()
self._has_up = "UP" in self.get_action_meanings()
self._horizon = int(horizon)
self.reset()
def __init__(self, game='pong', obs_type='ram'):
utils.EzPickle.__init__(self, game, obs_type)
assert obs_type in ('ram', 'image')
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
raise IOError('You asked for game %s but path %s does not exist' %
(game, self.game_path))
self._obs_type = obs_type
self.ale = atari_py.ALEInterface()
self.viewer = None
self._seed()
def test_smoke():
pong_path = atari_py.get_game_path('pong')
ale = atari_py.ALEInterface()
ale.loadROM(pong_path)
action_set = ale.getMinimalActionSet()
# Test stepping
ale.act(action_set[0])
# Test screen capture
(screen_width, screen_height) = ale.getScreenDims()
arr = np.zeros((screen_height, screen_width, 4), dtype=np.uint8)
ale.getScreenRGB(arr)
def __init__(self,
mode,
game='pong',
obs_type='image',
frameskip=(2, 5),
repeat_action_probability=0.,
full_action_space=False):
"""Frameskip should be either a tuple (indicating a random range to
choose from, with the top value exclude), or an int."""
utils.EzPickle.__init__(self, game, obs_type, frameskip,
repeat_action_probability)
assert obs_type in ('ram', 'image')
self.game = game
self.game_path = atari_py.get_game_path(game)
if not os.path.exists(self.game_path):
msg = 'You asked for game %s but path %s does not exist'
raise IOError(msg % (game, self.game_path))
self._obs_type = obs_type
self.frameskip = frameskip
self.ale = atari_py.ALEInterface()
self.viewer = None
# Tune (or disable) ALE's action repeat:
# https://github.com/openai/gym/issues/349
assert isinstance(repeat_action_probability, (float, int)), \
"Invalid repeat_action_probability: {!r}".format(repeat_action_probability)
self.ale.setFloat('repeat_action_probability'.encode('utf-8'),
repeat_action_probability)
self.seed(mode)
self._action_set = (self.ale.getLegalActionSet() if full_action_space
else self.ale.getMinimalActionSet())
self.action_space = spaces.Discrete(len(self._action_set))
# variable to change whether the environment uses RGB images or grayscale
self.image_type = 'rgb'
(screen_width, screen_height) = self.ale.getScreenDims()
if self._obs_type == 'ram':
self.observation_space = spaces.Box(low=0,
high=255,
dtype=np.uint8,
shape=(128, ))
elif self._obs_type == 'image':
self.observation_space = spaces.Box(low=0,
high=255,
shape=(screen_height,
screen_width, 3),
dtype=np.uint8)
else:
raise error.Error('Unrecognized observation type: {}'.format(
self._obs_type))
def __init__(self):
self.metadata = {'render.modes': ['human', "rgb_array"]}
# simulate environment to be like a game(pong)
self.ale = atari_py.ALEInterface()
self.game_path = atari_py.get_game_path('pong')
# initializing
self.number_of_words_to_trans = 0
self.ACTION = [['pick-up']]
self.phrase, self.target = None, None
self.in_production_mood = False
self.initial_for_reset = None
self.done = False
def __init__(
self,
game="pong",
frame_skip=4,
num_img_obs=4,
clip_reward=True,
episodic_lives=True,
max_start_noops=30,
repeat_action_probability=0.,
):
Serializable.quick_init(self, locals())
# ALE
game_path = atari_py.get_game_path(game)
if not os.path.exists(game_path):
raise IOError("You asked for game {} but path {} does not "
" exist".format(game, game_path))
self.ale = atari_py.ALEInterface()
self.ale.setFloat(b'repeat_action_probability',
repeat_action_probability)
self._repeat_action_probability = repeat_action_probability
self.ale.loadROM(game_path)
self._game = game
# Spaces
self._action_set = self.ale.getMinimalActionSet()
self._action_space = Discrete(len(self._action_set))
obs_shape = (num_img_obs, H, W)
self._observation_space = UintBox(shape=obs_shape, bits=8)
self._max_frame = self.ale.getScreenGrayscale()
self._raw_frame_1 = self._max_frame.copy()
self._raw_frame_2 = self._max_frame.copy()
self._obs = np.zeros(shape=obs_shape, dtype="uint8")
# Settings
self._frame_skip = frame_skip
self._num_img_obs = num_img_obs
self._clip_reward = clip_reward
self._max_start_noops = max_start_noops
self._has_fire = "FIRE" in self.get_action_meanings()
self._has_up = "UP" in self.get_action_meanings()
self._episodic_lives = episodic_lives
self._done = self._done_episodic_lives if episodic_lives else \
self._done_no_epidosic_lives
# Get ready
self.reset()
def __init__(
self,
game="pong", # 游戏名
frame_skip=4, # Frames per step (>=1).
num_img_obs=4, # Number of (past) frames in observation (>=1).
clip_reward=True,
episodic_lives=True,
max_start_noops=30,
repeat_action_probability=0.,
horizon=27000, # 人工随意定的一个值。在游戏角色没有死的时候，step大于这个值也会被判定为game over
):
"""
environment对象会在Sampler类中的initialize()方法里构造。
"""
save__init__args(
locals(), underscore=True
) # 非常tricky的做法：把局部变量保存到实例的属性中，之后如果找不到self.xxx的定义就在这里面找
# ALE，即电玩学习环境(Arcade Learning Environment)，它提供了一个关于Atari 2600游戏的数百个游戏环境的接口
game_path = atari_py.get_game_path(game)
if not os.path.exists(game_path):
raise IOError(
"You asked for game {} but path {} does not exist".format(
game, game_path))
self.ale = atari_py.ALEInterface()
self.ale.setFloat(b'repeat_action_probability',
repeat_action_probability)
self.ale.loadROM(game_path)
# Spaces，某些游戏可能不会对控制输入的所有组合都使用所有动作，例如pong游戏仅使用2个方向和射击键，这里要获取当前游戏使用的动作集
self._action_set = self.ale.getMinimalActionSet(
) # 最小的action set(一个numpy array)
self._action_space = IntBox(low=0, high=len(self._action_set))
obs_shape = (num_img_obs, H, W) # H应该是指height，W应该是指width
self._observation_space = IntBox(low=0,
high=255,
shape=obs_shape,
dtype="uint8")
self._max_frame = self.ale.getScreenGrayscale()
self._raw_frame_1 = self._max_frame.copy()
self._raw_frame_2 = self._max_frame.copy()
self._obs = np.zeros(shape=obs_shape, dtype="uint8") # 初始的observation
# Settings
self._has_fire = "FIRE" in self.get_action_meanings()
self._has_up = "UP" in self.get_action_meanings()
self._horizon = int(horizon)
self.reset()
def __init__(self, args):
self.device = args.device
self.ale = atari_py.ALEInterface()
self.ale.setInt('random_seed', args.seed)
self.ale.setInt('max_num_frames', args.max_episode_length)
self.ale.setFloat('repeat_action_probability', 0)
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
self.ale.loadROM(atari_py.get_game_path(args.game))
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e]
for i, e in zip(range(len(actions)), actions))
self.lives = 0
self.life_termination = False
self.window = args.history_length
self.state_buffer = deque([], maxlen=args.history_length)
self.training = True
def __init__(
self,
game="pong",
frame_shape=(80, 104),
frame_skip=4, # Frames per step (>=1).
num_img_obs=4, # Number of (past) frames in observation (>=1).
clip_reward=True,
episodic_lives=True,
max_start_noops=30,
repeat_action_probability=0.,
horizon=27000,
seed=0):
save__init__args(locals(), underscore=True)
# ALE
game_path = atari_py.get_game_path(game)
if not os.path.exists(game_path):
raise IOError("You asked for game {} but path {} does not "
" exist".format(game, game_path))
self.ale = atari_py.ALEInterface()
self.ale.setFloat(b'repeat_action_probability',
repeat_action_probability)
self.ale.loadROM(game_path)
# Spaces
self._action_set = self.ale.getMinimalActionSet()
self._action_space = IntBox(low=0, high=len(self._action_set))
self._frame_shape = frame_shape
obs_shape = (num_img_obs, frame_shape[1], frame_shape[0])
self._observation_space = IntBox(low=0,
high=255,
shape=obs_shape,
dtype="uint8")
self._max_frame = self.ale.getScreenGrayscale()
self._raw_frame_1 = self._max_frame.copy()
self._raw_frame_2 = self._max_frame.copy()
self._obs = np.zeros(shape=obs_shape, dtype="uint8")
self.random = np.random.RandomState(seed)
# Settings
self._has_fire = "FIRE" in self.get_action_meanings()
self._has_up = "UP" in self.get_action_meanings()
self._horizon = int(horizon)
self.reset()
def __init__(
self,
game="pong",
frame_skip=4,
num_img_obs=4,
clip_reward=True,
episodic_lives=False, # !
max_start_noops=30,
repeat_action_probability=0.25, # !
horizon=27000,
obs_size=84, # square resize
fire_on_reset=True,
):
save__init__args(locals(), underscore=True)
# ALE
game_path = atari_py.get_game_path(game)
if not os.path.exists(game_path):
raise IOError("You asked for game {} but path {} does not "
" exist".format(game, game_path))
self.ale = atari_py.ALEInterface()
self.ale.setFloat(b'repeat_action_probability',
repeat_action_probability)
self.ale.loadROM(game_path)
# Spaces
self._obs_size = obs_size
self._action_set = self.ale.getMinimalActionSet()
self._action_space = IntBox(low=0, high=len(self._action_set))
obs_shape = (num_img_obs, self._obs_size, self._obs_size)
self._observation_space = IntBox(low=0,
high=256,
shape=obs_shape,
dtype="uint8")
self._max_frame = self.ale.getScreenGrayscale()
self._raw_frame_1 = self._max_frame.copy()
self._raw_frame_2 = self._max_frame.copy()
self._obs = np.zeros(shape=obs_shape, dtype="uint8")
# Settings
self._has_fire = "FIRE" in self.get_action_meanings()
self._has_up = "UP" in self.get_action_meanings()
self._horizon = int(horizon)
self.reset()
def __init__(self, args):
# print("In init")
self.device = args.device
self.ale = atari_py.ALEInterface()
self.ale.setInt('random_seed', args.seed)
self.ale.setInt('max_num_frames_per_episode', args.max_episode_length)
self.ale.setFloat('repeat_action_probability', 0) # Disable sticky actions
self.ale.setInt('frame_skip', 0)
self.ale.setBool('color_averaging', False)
self.ale.loadROM(atari_py.get_game_path(args.game)) # ROM loading must be done after setting options
actions = self.ale.getMinimalActionSet()
self.actions = dict([i, e] for i, e in zip(range(len(actions)), actions))
self.lives = 0 # Life counter (used in DeepMind training)
self.life_termination = False # Used to check if resetting only from loss of life
self.window = args.history_length # Number of frames to concatenate
self.state_buffer = deque([], maxlen=args.history_length)
self.training = True # Consistent with model training mode
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.clip_model, self.preprocess = clip.load("ViT-B/32", device=self.device)
from episodic_agent import *
from ale_experiment import *
import atari_py
game_path = atari_py.get_game_path('pong')
ale = atari_py.ALEInterface()
#ale.setFloat('repeat_action_probability',0.0)
ale.loadROM(game_path)
num_actions = len(ale.getMinimalActionSet())
agent = EpisodicControlAgent(4)
experiment = ALEExperiment(ale,agent,84,84,'scale',5000,10000,0,4,True,30,rng = np.random.RandomState(123456))
experiment.run()