def _create_graph(self, task_parameters: TaskParameters) -> Tuple[List[LevelManager], List[Environment]]:
env = short_dynamic_import(self.env_params.path)(**self.env_params.__dict__,
visualization_parameters=self.visualization_parameters)
for agent_params in self.agents_params:
agent_params.task_parameters = task_parameters
# we need to build the hierarchy in reverse order (from the bottom up) in order for the spaces of each level
# to be known
level_managers = []
current_env = env
# out_action_space = env.action_space
for level_idx, agent_params in reversed(list(enumerate(self.agents_params))):
agent_params.name = "agent_{}".format(level_idx)
agent_params.is_a_highest_level_agent = level_idx == 0
agent_params.is_a_lowest_level_agent = level_idx == len(self.agents_params) - 1
agent = short_dynamic_import(agent_params.path)(agent_params)
level_manager = LevelManager(
agents=agent,
environment=current_env,
real_environment=env,
steps_limit=EnvironmentSteps(1) if level_idx == 0
else self.consecutive_steps_to_run_non_top_levels,
should_reset_agent_state_after_time_limit_passes=level_idx > 0,
name="level_{}".format(level_idx)
)
current_env = level_manager
level_managers.insert(0, level_manager)
return level_managers, [env]
def expand_preset(self, preset):
"""
Replace a short preset name with the full python path, and verify that it can be imported.
"""
if preset.lower() in [p.lower() for p in list_all_presets()]:
preset = "{}.py:graph_manager".format(
os.path.join(get_base_dir(), 'presets', preset))
else:
preset = "{}".format(preset)
# if a graph manager variable was not specified, try the default of :graph_manager
if len(preset.split(":")) == 1:
preset += ":graph_manager"
# verify that the preset exists
preset_path = preset.split(":")[0]
if not os.path.exists(preset_path):
screen.error(
"The given preset ({}) cannot be found.".format(preset))
# verify that the preset can be instantiated
try:
short_dynamic_import(preset, ignore_module_case=True)
except TypeError as e:
traceback.print_exc()
screen.error('Internal Error: ' + str(e) +
"\n\nThe given preset ({}) cannot be instantiated.".
format(preset))
return preset
def _create_graph(
self, task_parameters: TaskParameters
) -> Tuple[List[MultiAgentLevelManager], List[Environment]]:
# environment loading
self.env_params.seed = task_parameters.seed
self.env_params.experiment_path = task_parameters.experiment_path
env = short_dynamic_import(self.env_params.path)(
**self.env_params.__dict__,
visualization_parameters=self.visualization_parameters)
# agent loading
agents = OrderedDict()
for agent_params in self.agents_params:
agent_params.task_parameters = copy.copy(task_parameters)
agent = short_dynamic_import(agent_params.path)(agent_params)
agents[agent_params.name] = agent
screen.log_title("Created agent: {}".format(agent_params.name))
if hasattr(self, 'memory_backend_params') and \
self.memory_backend_params.run_type == str(RunType.ROLLOUT_WORKER):
agent.memory.memory_backend = deepracer_memory.DeepRacerRolloutBackEnd(
self.memory_backend_params,
agent_params.algorithm.num_consecutive_playing_steps,
agent_params.name)
# set level manager
level_manager = MultiAgentLevelManager(
agents=agents,
environment=env,
name="main_level",
done_condition=self.done_condition)
return [level_manager], [env]
def _create_graph(self, task_parameters: TaskParameters) -> Tuple[List[LevelManager], List[Environment]]:
if self.env_params:
# environment loading
self.env_params.seed = task_parameters.seed
self.env_params.experiment_path = task_parameters.experiment_path
env = short_dynamic_import(self.env_params.path)(**self.env_params.__dict__,
visualization_parameters=self.visualization_parameters)
else:
env = None
# Only DQN variants and NEC are supported at this point.
assert(isinstance(self.agent_params, DQNAgentParameters) or isinstance(self.agent_params, NECAgentParameters))
# Only Episodic memories are supported,
# for evaluating the sequential doubly robust estimator
assert(isinstance(self.agent_params.memory, EpisodicExperienceReplayParameters))
# agent loading
self.agent_params.task_parameters = task_parameters # TODO: this should probably be passed in a different way
self.agent_params.name = "agent"
self.agent_params.is_batch_rl_training = True
self.agent_params.network_wrappers['main'].should_get_softmax_probabilities = True
if 'reward_model' not in self.agent_params.network_wrappers:
# user hasn't defined params for the reward model. we will use the same params as used for the 'main'
# network.
self.agent_params.network_wrappers['reward_model'] = deepcopy(self.agent_params.network_wrappers['main'])
self.agent = short_dynamic_import(self.agent_params.path)(self.agent_params)
agents = {'agent': self.agent}
if not self.is_collecting_random_dataset:
self.experience_generating_agent_params.visualization.dump_csv = False
self.experience_generating_agent_params.task_parameters = task_parameters
self.experience_generating_agent_params.name = "experience_gen_agent"
self.experience_generating_agent_params.network_wrappers['main'].should_get_softmax_probabilities = True
# we need to set these manually as these are usually being set for us only for the default agent
self.experience_generating_agent_params.input_filter = self.agent_params.input_filter
self.experience_generating_agent_params.output_filter = self.agent_params.output_filter
self.experience_generating_agent = short_dynamic_import(
self.experience_generating_agent_params.path)(self.experience_generating_agent_params)
agents['experience_generating_agent'] = self.experience_generating_agent
if not env and not self.agent_params.memory.load_memory_from_file_path:
screen.warning("A BatchRLGraph requires setting a dataset to load into the agent's memory or alternatively "
"using an environment to create a (random) dataset from. This agent should only be used for "
"inference. ")
# set level manager
# - although we will be using each agent separately, we have to have both agents initialized together with the
# LevelManager, so to have them both properly initialized
level_manager = LevelManager(agents=agents,
environment=env, name="main_level",
spaces_definition=self.spaces_definition)
if env:
return [level_manager], [env]
else:
return [level_manager], []
def get_graph_manager_from_args(self, args: argparse.Namespace) -> 'GraphManager':
"""
Return the graph manager according to the command line arguments given by the user.
:param args: the arguments given by the user
:return: the graph manager, not bound to task_parameters yet.
"""
graph_manager = None
# if a preset was given we will load the graph manager for the preset
if args.preset is not None:
graph_manager = short_dynamic_import(args.preset, ignore_module_case=True)
# for human play we need to create a custom graph manager
if args.play:
from rl_coach.agents.human_agent import HumanAgentParameters
env_params = short_dynamic_import(args.environment_type, ignore_module_case=True)()
env_params.human_control = True
schedule_params = HumanPlayScheduleParameters()
graph_manager = BasicRLGraphManager(HumanAgentParameters(), env_params, schedule_params, VisualizationParameters())
# Set framework
# Note: Some graph managers (e.g. HAC preset) create multiple agents and the attribute is called agents_params
if hasattr(graph_manager, 'agent_params'):
for network_parameters in graph_manager.agent_params.network_wrappers.values():
network_parameters.framework = args.framework
elif hasattr(graph_manager, 'agents_params'):
for ap in graph_manager.agents_params:
for network_parameters in ap.network_wrappers.values():
network_parameters.framework = args.framework
if args.level:
if isinstance(graph_manager.env_params.level, SingleLevelSelection):
graph_manager.env_params.level.select(args.level)
else:
graph_manager.env_params.level = args.level
# set the seed for the environment
if args.seed is not None:
graph_manager.env_params.seed = args.seed
# visualization
graph_manager.visualization_parameters.dump_gifs = graph_manager.visualization_parameters.dump_gifs or args.dump_gifs
graph_manager.visualization_parameters.dump_mp4 = graph_manager.visualization_parameters.dump_mp4 or args.dump_mp4
graph_manager.visualization_parameters.render = args.render
graph_manager.visualization_parameters.tensorboard = args.tensorboard
graph_manager.visualization_parameters.print_networks_summary = args.print_networks_summary
# update the custom parameters
if args.custom_parameter is not None:
unstripped_key_value_pairs = [pair.split('=') for pair in args.custom_parameter.split(';')]
stripped_key_value_pairs = [tuple([pair[0].strip(), pair[1].strip()]) for pair in
unstripped_key_value_pairs if len(pair) == 2]
# load custom parameters into run_dict
for key, value in stripped_key_value_pairs:
exec("graph_manager.{}={}".format(key, value))
return graph_manager
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--markov-preset-file',
help="(string) Name of a preset file to run in Markov's preset directory.",
type=str,
default=os.environ.get("MARKOV_PRESET_FILE", "object_tracker.py"))
parser.add_argument('--model-s3-bucket',
help='(string) S3 bucket where trained models are stored. It contains model checkpoints.',
type=str,
default=os.environ.get("MODEL_S3_BUCKET"))
parser.add_argument('--model-s3-prefix',
help='(string) S3 prefix where trained models are stored. It contains model checkpoints.',
type=str,
default=os.environ.get("MODEL_S3_PREFIX"))
parser.add_argument('--aws-region',
help='(string) AWS region',
type=str,
default=os.environ.get("ROS_AWS_REGION", "us-west-2"))
parser.add_argument('--number-of-trials',
help='(integer) Number of trials',
type=int,
default=os.environ.get("NUMBER_OF_TRIALS", sys.maxsize))
parser.add_argument('-c', '--local-model-directory',
help='(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default='./checkpoint')
args = parser.parse_args()
data_store_params_instance = S3BotoDataStoreParameters(bucket_name=args.model_s3_bucket,
s3_folder=args.model_s3_prefix,
checkpoint_dir=args.local_model_directory,
aws_region=args.aws_region)
data_store = S3BotoDataStore(data_store_params_instance)
utils.wait_for_checkpoint(args.local_model_directory, data_store)
preset_file_success = data_store.download_presets_if_present(PRESET_LOCAL_PATH)
if preset_file_success:
environment_file_success = data_store.download_environments_if_present(ENVIRONMENT_LOCAL_PATH)
path_and_module = PRESET_LOCAL_PATH + args.markov_preset_file + ":graph_manager"
graph_manager = short_dynamic_import(path_and_module, ignore_module_case=True)
if environment_file_success:
import robomaker.environments
print("Using custom preset file!")
elif args.markov_preset_file:
markov_path = imp.find_module("markov")[1]
preset_location = os.path.join(markov_path, "presets", args.markov_preset_file)
path_and_module = preset_location + ":graph_manager"
graph_manager = short_dynamic_import(path_and_module, ignore_module_case=True)
print("Using custom preset file from Markov presets directory!")
else:
raise ValueError("Unable to determine preset file")
graph_manager.data_store = data_store
evaluation_worker(
graph_manager=graph_manager,
number_of_trials=args.number_of_trials,
local_model_directory=args.local_model_directory
)
def get_graph_manager_from_args(args: argparse.Namespace) -> 'GraphManager':
"""
Return the graph manager according to the command line arguments given by the user
:param args: the arguments given by the user
:return: the updated graph manager
"""
graph_manager = None
# if a preset was given we will load the graph manager for the preset
if args.preset is not None:
graph_manager = short_dynamic_import(args.preset,
ignore_module_case=True)
# for human play we need to create a custom graph manager
if args.play:
env_params = short_dynamic_import(args.environment_type,
ignore_module_case=True)()
env_params.human_control = True
schedule_params = HumanPlayScheduleParameters()
graph_manager = BasicRLGraphManager(HumanAgentParameters(), env_params,
schedule_params,
VisualizationParameters())
if args.level:
if isinstance(graph_manager.env_params.level, SingleLevelSelection):
graph_manager.env_params.level.select(args.level)
else:
graph_manager.env_params.level = args.level
# set the seed for the environment
if args.seed is not None:
graph_manager.env_params.seed = args.seed
# visualization
graph_manager.visualization_parameters.dump_gifs = graph_manager.visualization_parameters.dump_gifs or args.dump_gifs
graph_manager.visualization_parameters.dump_mp4 = graph_manager.visualization_parameters.dump_mp4 or args.dump_mp4
graph_manager.visualization_parameters.render = args.render
graph_manager.visualization_parameters.tensorboard = args.tensorboard
# update the custom parameters
if args.custom_parameter is not None:
unstripped_key_value_pairs = [
pair.split('=') for pair in args.custom_parameter.split(';')
]
stripped_key_value_pairs = [
tuple([pair[0].strip(), pair[1].strip()])
for pair in unstripped_key_value_pairs if len(pair) == 2
]
# load custom parameters into run_dict
for key, value in stripped_key_value_pairs:
exec("graph_manager.{}={}".format(key, value))
return graph_manager
def _create_graph(
self, task_parameters: TaskParameters
) -> Tuple[List[LevelManager], List[Environment]]:
self.env_params.seed = task_parameters.seed
env = short_dynamic_import(self.env_params.path)(
**self.env_params.__dict__,
visualization_parameters=self.visualization_parameters)
for agent_params in self.agents_params:
agent_params.task_parameters = task_parameters
# we need to build the hierarchy in reverse order (from the bottom up) in order for the spaces of each level
# to be known
level_managers = []
current_env = env
# out_action_space = env.action_space
for level_idx, agent_params in reversed(
list(enumerate(self.agents_params))):
# TODO: the code below is specific for HRL on observation scale
# in action space
# if level_idx == 0:
# # top level agents do not get directives
# in_action_space = None
# else:
# pass
# attention_size = (env.state_space['observation'].shape - 1)//4
# in_action_space = AttentionActionSpace(shape=2, low=0, high=env.state_space['observation'].shape - 1,
# forced_attention_size=attention_size)
# agent_params.output_filter.action_filters['masking'].set_masking(0, attention_size)
agent_params.name = "agent_{}".format(level_idx)
agent_params.is_a_highest_level_agent = level_idx == 0
agent = short_dynamic_import(agent_params.path)(agent_params)
level_manager = LevelManager(
agents=agent,
environment=current_env,
real_environment=env,
steps_limit=self.
consecutive_steps_to_run_each_level[level_idx],
should_reset_agent_state_after_time_limit_passes=level_idx > 0,
name="level_{}".format(level_idx))
current_env = level_manager
level_managers.insert(0, level_manager)
# out_action_space = in_action_space
return level_managers, [env]
def _create_graph(
self, task_parameters: TaskParameters
) -> Tuple[List[LevelManager], List[Environment]]:
if self.env_params:
# environment loading
self.env_params.seed = task_parameters.seed
self.env_params.experiment_path = task_parameters.experiment_path
env = short_dynamic_import(self.env_params.path)(
**self.env_params.__dict__,
visualization_parameters=self.visualization_parameters)
else:
env = None
# Only DQN variants and NEC are supported at this point.
assert (isinstance(self.agent_params, DQNAgentParameters)
or isinstance(self.agent_params, NECAgentParameters))
# Only Episodic memories are supported,
# for evaluating the sequential doubly robust estimator
assert (isinstance(self.agent_params.memory,
EpisodicExperienceReplayParameters))
# agent loading
self.agent_params.task_parameters = task_parameters # TODO: this should probably be passed in a different way
self.agent_params.name = "agent"
self.agent_params.is_batch_rl_training = True
if 'reward_model' not in self.agent_params.network_wrappers:
# user hasn't defined params for the reward model. we will use the same params as used for the 'main'
# network.
self.agent_params.network_wrappers['reward_model'] = deepcopy(
self.agent_params.network_wrappers['main'])
agent = short_dynamic_import(self.agent_params.path)(self.agent_params)
if not env and not self.agent_params.memory.load_memory_from_file_path:
screen.warning(
"A BatchRLGraph requires setting a dataset to load into the agent's memory or alternatively "
"using an environment to create a (random) dataset from. This agent should only be used for "
"inference. ")
# set level manager
level_manager = LevelManager(agents=agent,
environment=env,
name="main_level",
spaces_definition=self.spaces_definition)
if env:
return [level_manager], [env]
else:
return [level_manager], []
def _create_graph(self, task_parameters: TaskParameters) -> Tuple[List[LevelManager], List[Environment]]:
# environment loading
self.env_params.seed = task_parameters.seed
env = short_dynamic_import(self.env_params.path)(**self.env_params.__dict__,
visualization_parameters=self.visualization_parameters)
# agent loading
self.agent_params.task_parameters = task_parameters # TODO: this should probably be passed in a different way
self.agent_params.name = "agent"
agent = short_dynamic_import(self.agent_params.path)(self.agent_params)
# set level manager
level_manager = LevelManager(agents=agent, environment=env, name="main_level")
return [level_manager], [env]
def preset_from_name(self, preset_name):
preset_path = self.path_of_main_launcher()
print("Loading preset %s from %s" % (preset_name, preset_path))
preset_path = os.path.join(self.path_of_main_launcher(),
preset_name) + ".py:graph_manager"
graph_manager = short_dynamic_import(preset_path,
ignore_module_case=True)
return graph_manager
def create_agents(self):
for agent_name, agent_parameters in self.agents_parameters.items():
agent_parameters.name = agent_name
# create agent
self.agents[agent_parameters.name] = short_dynamic_import(agent_parameters.path)(agent_parameters,
parent=self)
self.agents[agent_parameters.name].parent_level_manager = self.parent_level_manager
# TODO: this is a bit too specific to be defined here
# add an attention cropping filter if the incoming directives are attention boxes
if isinstance(self.in_action_space, AttentionActionSpace):
attention_size = self.in_action_space.forced_attention_size
for agent in self.agents.values():
agent.input_filter.observation_filters['attention'] = \
ObservationCropFilter(crop_low=np.zeros_like(attention_size), crop_high=attention_size)
agent.input_filter.observation_filters.move_to_end('attention', last=False) # add the cropping at the beginning
def __init__(self,
level: LevelSelection,
frame_skip: int,
visualization_parameters: VisualizationParameters,
target_success_rate: float = 1.0,
additional_simulator_parameters: Dict[str, Any] = {},
seed: Union[None, int] = None,
human_control: bool = False,
custom_reward_threshold: Union[int, float] = None,
random_initialization_steps: int = 1,
max_over_num_frames: int = 1,
observation_space_type: ObservationSpaceType = None,
**kwargs):
"""
:param level: (str)
A string representing the gym level to run. This can also be a LevelSelection object.
For example, BreakoutDeterministic-v0
:param frame_skip: (int)
The number of frames to skip between any two actions given by the agent. The action will be repeated
for all the skipped frames.
:param visualization_parameters: (VisualizationParameters)
The parameters used for visualizing the environment, such as the render flag, storing videos etc.
:param additional_simulator_parameters: (Dict[str, Any])
Any additional parameters that the user can pass to the Gym environment. These parameters should be
accepted by the __init__ function of the implemented Gym environment.
:param seed: (int)
A seed to use for the random number generator when running the environment.
:param human_control: (bool)
A flag that allows controlling the environment using the keyboard keys.
:param custom_reward_threshold: (float)
Allows defining a custom reward that will be used to decide when the agent succeeded in passing the environment.
If not set, this value will be taken from the Gym environment definition.
:param random_initialization_steps: (int)
The number of random steps that will be taken in the environment after each reset.
This is a feature presented in the DQN paper, which improves the variability of the episodes the agent sees.
:param max_over_num_frames: (int)
This value will be used for merging multiple frames into a single frame by taking the maximum value for each
of the pixels in the frame. This is particularly used in Atari games, where the frames flicker, and objects
can be seen in one frame but disappear in the next.
:param observation_space_type:
This value will be used for generating observation space. Allows a custom space. Should be one of
ObservationSpaceType. If not specified, observation space is inferred from the number of dimensions
of the observation: 1D: Vector space, 3D: Image space if 1 or 3 channels, PlanarMaps space otherwise.
"""
super().__init__(level, seed, frame_skip, human_control,
custom_reward_threshold, visualization_parameters,
target_success_rate)
self.random_initialization_steps = random_initialization_steps
self.max_over_num_frames = max_over_num_frames
self.additional_simulator_parameters = additional_simulator_parameters
# hide warnings
gym.logger.set_level(40)
"""
load and initialize environment
environment ids can be defined in 3 ways:
1. Native gym environments like BreakoutDeterministic-v0 for example
2. Custom gym environments written and installed as python packages.
This environments should have a python module with a class inheriting gym.Env, implementing the
relevant functions (_reset, _step, _render) and defining the observation and action space
For example: my_environment_package:MyEnvironmentClass will run an environment defined in the
MyEnvironmentClass class
3. Custom gym environments written as an independent module which is not installed.
This environments should have a python module with a class inheriting gym.Env, implementing the
relevant functions (_reset, _step, _render) and defining the observation and action space.
For example: path_to_my_environment.sub_directory.my_module:MyEnvironmentClass will run an
environment defined in the MyEnvironmentClass class which is located in the module in the relative path
path_to_my_environment.sub_directory.my_module
"""
if ':' in self.env_id:
# custom environments
if '/' in self.env_id or '.' in self.env_id:
# environment in a an absolute path module written as a unix path or in a relative path module
# written as a python import path
env_class = short_dynamic_import(self.env_id)
else:
# environment in a python package
env_class = gym.envs.registration.load(self.env_id)
# instantiate the environment
try:
self.env = env_class(**self.additional_simulator_parameters)
except:
screen.error(
"Failed to instantiate Gym environment class %s with arguments %s"
% (env_class, self.additional_simulator_parameters),
crash=False)
raise
else:
self.env = gym.make(self.env_id)
# for classic control we want to use the native renderer because otherwise we will get 2 renderer windows
environment_to_always_use_with_native_rendering = [
'classic_control', 'mujoco', 'robotics'
]
self.native_rendering = self.native_rendering or \
any([env in str(self.env.unwrapped.__class__)
for env in environment_to_always_use_with_native_rendering])
if self.native_rendering:
if hasattr(self, 'renderer'):
self.renderer.close()
# seed
if self.seed is not None:
self.env.seed(self.seed)
np.random.seed(self.seed)
random.seed(self.seed)
# frame skip and max between consecutive frames
self.is_mujoco_env = 'mujoco' in str(self.env.unwrapped.__class__)
self.is_roboschool_env = 'roboschool' in str(
self.env.unwrapped.__class__)
self.is_atari_env = 'Atari' in str(self.env.unwrapped.__class__)
if self.is_atari_env:
self.env.unwrapped.frameskip = 1 # this accesses the atari env that is wrapped with a timelimit wrapper env
if self.env_id == "SpaceInvadersDeterministic-v4" and self.frame_skip == 4:
screen.warning(
"Warning: The frame-skip for Space Invaders was automatically updated from 4 to 3. "
"This is following the DQN paper where it was noticed that a frame-skip of 3 makes the "
"laser rays disappear. To force frame-skip of 4, please use SpaceInvadersNoFrameskip-v4."
)
self.frame_skip = 3
self.env = MaxOverFramesAndFrameskipEnvWrapper(
self.env,
frameskip=self.frame_skip,
max_over_num_frames=self.max_over_num_frames)
else:
self.env.unwrapped.frameskip = self.frame_skip
self.state_space = StateSpace({})
# observations
if not isinstance(self.env.observation_space, gym.spaces.dict.Dict):
state_space = {'observation': self.env.observation_space}
else:
state_space = self.env.observation_space.spaces
for observation_space_name, observation_space in state_space.items():
if observation_space_type == ObservationSpaceType.Tensor:
# we consider arbitrary input tensor which does not necessarily represent images
self.state_space[
observation_space_name] = TensorObservationSpace(
shape=np.array(observation_space.shape),
low=observation_space.low,
high=observation_space.high)
elif observation_space_type == ObservationSpaceType.Image or len(
observation_space.shape) == 3:
# we assume gym has image observations (with arbitrary number of channels) where their values are
# within 0-255, and where the channel dimension is the last dimension
if observation_space.shape[-1] in [1, 3]:
self.state_space[
observation_space_name] = ImageObservationSpace(
shape=np.array(observation_space.shape),
high=255,
channels_axis=-1)
else:
# For any number of channels other than 1 or 3, use the generic PlanarMaps space
self.state_space[
observation_space_name] = PlanarMapsObservationSpace(
shape=np.array(observation_space.shape),
low=0,
high=255,
channels_axis=-1)
elif observation_space_type == ObservationSpaceType.Vector or len(
observation_space.shape) == 1:
self.state_space[
observation_space_name] = VectorObservationSpace(
shape=observation_space.shape[0],
low=observation_space.low,
high=observation_space.high)
else:
raise screen.error(
"Failed to instantiate Gym environment class %s with observation space type %s"
% (env_class, observation_space_type),
crash=True)
if 'desired_goal' in state_space.keys():
self.goal_space = self.state_space['desired_goal']
# actions
if type(self.env.action_space) == gym.spaces.box.Box:
self.action_space = BoxActionSpace(
shape=self.env.action_space.shape,
low=self.env.action_space.low,
high=self.env.action_space.high)
elif type(self.env.action_space) == gym.spaces.discrete.Discrete:
actions_description = []
if hasattr(self.env.unwrapped, 'get_action_meanings'):
actions_description = self.env.unwrapped.get_action_meanings()
self.action_space = DiscreteActionSpace(
num_actions=self.env.action_space.n,
descriptions=actions_description)
else:
raise screen.error((
"Failed to instantiate gym environment class {} due to unsupported "
"action space {}. Expected BoxActionSpace or DiscreteActionSpace."
).format(env_class, self.env.action_space),
crash=True)
if self.human_control:
# TODO: add this to the action space
# map keyboard keys to actions
self.key_to_action = {}
if hasattr(self.env.unwrapped, 'get_keys_to_action'):
self.key_to_action = self.env.unwrapped.get_keys_to_action()
else:
screen.error(
"Error: Environment {} does not support human control.".
format(self.env),
crash=True)
# initialize the state by getting a new state from the environment
self.reset_internal_state(True)
# render
if self.is_rendered:
image = self.get_rendered_image()
scale = 1
if self.human_control:
scale = 2
if not self.native_rendering:
self.renderer.create_screen(image.shape[1] * scale,
image.shape[0] * scale)
# the info is only updated after the first step
self.state = self.step(self.action_space.default_action).next_state
self.state_space['measurements'] = VectorObservationSpace(
shape=len(self.info.keys()))
if self.env.spec and custom_reward_threshold is None:
self.reward_success_threshold = self.env.spec.reward_threshold
self.reward_space = RewardSpace(
1, reward_success_threshold=self.reward_success_threshold)
self.target_success_rate = target_success_rate
def main():
screen.set_use_colors(False)
logger.info("src/training_worker.py - INIZIO MAIN")
parser = argparse.ArgumentParser()
parser.add_argument('-pk',
'--preset_s3_key',
help="(string) Name of a preset to download from S3",
type=str,
required=False)
parser.add_argument(
'-ek',
'--environment_s3_key',
help="(string) Name of an environment file to download from S3",
type=str,
required=False)
parser.add_argument('--model_metadata_s3_key',
help="(string) Model Metadata File S3 Key",
type=str,
required=False)
parser.add_argument(
'-c',
'--checkpoint-dir',
help=
'(string) Path to a folder containing a checkpoint to write the model to.',
type=str,
default='./checkpoint')
parser.add_argument(
'--pretrained-checkpoint-dir',
help='(string) Path to a folder for downloading a pre-trained model',
type=str,
default=PRETRAINED_MODEL_DIR)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get(
"SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default='sagemaker')
parser.add_argument('--s3_endpoint_url',
help='(string) S3 endpoint URL',
type=str,
default=os.environ.get("S3_ENDPOINT_URL", None))
parser.add_argument('--framework',
help='(string) tensorflow or mxnet',
type=str,
default='tensorflow')
parser.add_argument('--pretrained_s3_bucket',
help='(string) S3 bucket for pre-trained model',
type=str)
parser.add_argument('--pretrained_s3_prefix',
help='(string) S3 prefix for pre-trained model',
type=str,
default='sagemaker')
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("AWS_REGION", "us-east-1"))
args, _ = parser.parse_known_args()
logger.info("S3 bucket: %s \n S3 prefix: %s \n S3 endpoint URL: %s",
args.s3_bucket, args.s3_prefix, args.s3_endpoint_url)
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region,
s3_endpoint_url=args.s3_endpoint_url)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
utils.load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
s3_client.upload_file(
os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix),
model_metadata_local_path)
shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR)
success_custom_preset = False
if args.preset_s3_key:
preset_local_path = "./markov/presets/preset.py"
success_custom_preset = s3_client.download_file(
s3_key=args.preset_s3_key, local_path=preset_local_path)
if not success_custom_preset:
logger.info(
"Could not download the preset file. Using the default DeepRacer preset."
)
else:
preset_location = "markov.presets.preset:graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
success_custom_preset = s3_client.upload_file(
s3_key=os.path.normpath("%s/presets/preset.py" %
args.s3_prefix),
local_path=preset_local_path)
if success_custom_preset:
logger.info("Using preset: %s" % args.preset_s3_key)
if not success_custom_preset:
params_blob = os.environ.get('SM_TRAINING_ENV', '')
if params_blob:
params = json.loads(params_blob)
sm_hyperparams_dict = params["hyperparameters"]
else:
sm_hyperparams_dict = {}
#configurazione agente: metadati del modello impostati dall'utente (angolo di sterzo + velocità ) + nome
#! TODO each agent should have own config
agent_config = {
'model_metadata': model_metadata_local_path,
ConfigParams.CAR_CTRL_CONFIG.value: {
ConfigParams.LINK_NAME_LIST.value: [],
ConfigParams.VELOCITY_LIST.value: {},
ConfigParams.STEERING_LIST.value: {},
ConfigParams.CHANGE_START.value: None,
ConfigParams.ALT_DIR.value: None,
ConfigParams.ACTION_SPACE_PATH.value:
'custom_files/model_metadata.json',
ConfigParams.REWARD.value: None,
ConfigParams.AGENT_NAME.value: 'racecar'
}
}
agent_list = list()
agent_list.append(create_training_agent(agent_config))
logger.info(
"src/training_worker.py - ora chiamo la get_graph_manager, che recupera l'agente"
)
graph_manager, robomaker_hyperparams_json = get_graph_manager(
hp_dict=sm_hyperparams_dict,
agent_list=agent_list,
run_phase_subject=None)
logger.info("src/training_worker.py - ho l'agente")
s3_client.upload_hyperparameters(robomaker_hyperparams_json)
logger.info("Uploaded hyperparameters.json to S3")
# Attach sample collector to graph_manager only if sample count > 0
max_sample_count = int(sm_hyperparams_dict.get("max_sample_count", 0))
if max_sample_count > 0:
sample_collector = SampleCollector(
s3_client=s3_client,
s3_prefix=args.s3_prefix,
max_sample_count=max_sample_count,
sampling_frequency=int(
sm_hyperparams_dict.get("sampling_frequency", 1)))
graph_manager.sample_collector = sample_collector
host_ip_address = utils.get_ip_from_host()
s3_client.write_ip_config(host_ip_address)
logger.info("Uploaded IP address information to S3: %s" % host_ip_address)
use_pretrained_model = args.pretrained_s3_bucket and args.pretrained_s3_prefix
# Handle backward compatibility
_, network_type, version = parse_model_metadata(model_metadata_local_path)
if use_pretrained_model:
if float(version) < float(SIMAPP_VERSION) and \
not utils.has_current_ckpnt_name(args.pretrained_s3_bucket, args.pretrained_s3_prefix, args.aws_region, args.s3_endpoint_url):
utils.make_compatible(args.pretrained_s3_bucket,
args.pretrained_s3_prefix, args.aws_region,
SyncFiles.TRAINER_READY.value)
#Select the optimal model for the starting weights
utils.do_model_selection(s3_bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
region=args.aws_region,
s3_endpoint_url=args.s3_endpoint_url)
ds_params_instance_pretrained = S3BotoDataStoreParameters(
aws_region=args.aws_region,
bucket_names={'agent': args.pretrained_s3_bucket},
base_checkpoint_dir=args.pretrained_checkpoint_dir,
s3_folders={'agent': args.pretrained_s3_prefix},
s3_endpoint_url=args.s3_endpoint_url)
data_store_pretrained = S3BotoDataStore(ds_params_instance_pretrained,
graph_manager, True)
data_store_pretrained.load_from_store()
memory_backend_params = DeepRacerRedisPubSubMemoryBackendParameters(
redis_address="localhost",
redis_port=6379,
run_type=str(RunType.TRAINER),
channel=args.s3_prefix,
network_type=network_type)
graph_manager.memory_backend_params = memory_backend_params
ds_params_instance = S3BotoDataStoreParameters(
aws_region=args.aws_region,
bucket_names={'agent': args.s3_bucket},
base_checkpoint_dir=args.checkpoint_dir,
s3_folders={'agent': args.s3_prefix},
s3_endpoint_url=args.s3_endpoint_url)
graph_manager.data_store_params = ds_params_instance
graph_manager.data_store = S3BotoDataStore(ds_params_instance,
graph_manager)
task_parameters = TaskParameters()
task_parameters.experiment_path = SM_MODEL_OUTPUT_DIR
task_parameters.checkpoint_save_secs = 20
if use_pretrained_model:
task_parameters.checkpoint_restore_path = args.pretrained_checkpoint_dir
task_parameters.checkpoint_save_dir = args.checkpoint_dir
#funzione riga 48
#prende in input:
# - il grafo (creato con la get_graph_manager)
# - robomaker_hyperparams_json (ritornato dalla get_graph_manager)
training_worker(
graph_manager=graph_manager,
task_parameters=task_parameters,
user_batch_size=json.loads(robomaker_hyperparams_json)["batch_size"],
user_episode_per_rollout=json.loads(
robomaker_hyperparams_json)["num_episodes_between_training"])
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument('-pk', '--preset_s3_key',
help="(string) Name of a preset to download from S3",
type=str,
required=False)
parser.add_argument('-ek', '--environment_s3_key',
help="(string) Name of an environment file to download from S3",
type=str,
required=False)
parser.add_argument('--model_metadata_s3_key',
help="(string) Model Metadata File S3 Key",
type=str,
required=False)
parser.add_argument('-c', '--checkpoint-dir',
help='(string) Path to a folder containing a checkpoint to write the model to.',
type=str,
default='./checkpoint')
parser.add_argument('--pretrained-checkpoint-dir',
help='(string) Path to a folder for downloading a pre-trained model',
type=str,
default=PRETRAINED_MODEL_DIR)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get("SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default='sagemaker')
parser.add_argument('--framework',
help='(string) tensorflow or mxnet',
type=str,
default='tensorflow')
parser.add_argument('--pretrained_s3_bucket',
help='(string) S3 bucket for pre-trained model',
type=str)
parser.add_argument('--pretrained_s3_prefix',
help='(string) S3 prefix for pre-trained model',
type=str,
default='sagemaker')
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("AWS_REGION", "us-east-1"))
start_redis_server()
args, _ = parser.parse_known_args()
s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH, 'model_metadata.json')
utils.load_model_metadata(s3_client, args.model_metadata_s3_key, model_metadata_local_path)
s3_client.upload_file(os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix), model_metadata_local_path)
shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR)
success_custom_preset = False
if args.preset_s3_key:
preset_local_path = "./markov/presets/preset.py"
success_custom_preset = s3_client.download_file(s3_key=args.preset_s3_key, local_path=preset_local_path)
if not success_custom_preset:
logger.info("Could not download the preset file. Using the default DeepRacer preset.")
else:
preset_location = "markov.presets.preset:graph_manager"
graph_manager = short_dynamic_import(preset_location, ignore_module_case=True)
success_custom_preset = s3_client.upload_file(
s3_key=os.path.normpath("%s/presets/preset.py" % args.s3_prefix), local_path=preset_local_path)
if success_custom_preset:
logger.info("Using preset: %s" % args.preset_s3_key)
if not success_custom_preset:
params_blob = os.environ.get('SM_TRAINING_ENV', '')
if params_blob:
params = json.loads(params_blob)
sm_hyperparams_dict = params["hyperparameters"]
else:
sm_hyperparams_dict = {}
#! TODO each agent should have own config
agent_config = {'model_metadata': model_metadata_local_path,
'car_ctrl_cnfig': {ConfigParams.LINK_NAME_LIST.value: [],
ConfigParams.VELOCITY_LIST.value : {},
ConfigParams.STEERING_LIST.value : {},
ConfigParams.CHANGE_START.value : None,
ConfigParams.ALT_DIR.value : None,
ConfigParams.ACTION_SPACE_PATH.value : 'custom_files/model_metadata.json',
ConfigParams.REWARD.value : None,
ConfigParams.AGENT_NAME.value : 'racecar'}}
agent_list = list()
agent_list.append(create_training_agent(agent_config))
#agent_list.append(create_training_agent(agent_config))
graph_manager, robomaker_hyperparams_json = get_graph_manager(sm_hyperparams_dict, agent_list)
s3_client.upload_hyperparameters(robomaker_hyperparams_json)
logger.info("Uploaded hyperparameters.json to S3")
host_ip_address = utils.get_ip_from_host()
s3_client.write_ip_config(host_ip_address)
logger.info("Uploaded IP address information to S3: %s" % host_ip_address)
use_pretrained_model = args.pretrained_s3_bucket and args.pretrained_s3_prefix
if use_pretrained_model:
# Handle backward compatibility
_, _, version = parse_model_metadata(model_metadata_local_path)
if float(version) < float(utils.SIMAPP_VERSION) and \
not utils.has_current_ckpnt_name(args.pretrained_s3_bucket, args.pretrained_s3_prefix, args.aws_region):
utils.make_compatible(args.pretrained_s3_bucket, args.pretrained_s3_prefix,
args.aws_region, SyncFiles.TRAINER_READY.value)
ds_params_instance_pretrained = S3BotoDataStoreParameters(aws_region=args.aws_region,
bucket_name=args.pretrained_s3_bucket,
checkpoint_dir=args.pretrained_checkpoint_dir,
s3_folder=args.pretrained_s3_prefix)
data_store_pretrained = S3BotoDataStore(ds_params_instance_pretrained)
data_store_pretrained.load_from_store()
memory_backend_params = RedisPubSubMemoryBackendParameters(redis_address="localhost",
redis_port=6379,
run_type=str(RunType.TRAINER),
channel=args.s3_prefix)
graph_manager.memory_backend_params = memory_backend_params
ds_params_instance = S3BotoDataStoreParameters(aws_region=args.aws_region,
bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
s3_folder=args.s3_prefix)
graph_manager.data_store_params = ds_params_instance
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
task_parameters = TaskParameters()
task_parameters.experiment_path = SM_MODEL_OUTPUT_DIR
task_parameters.checkpoint_save_secs = 20
if use_pretrained_model:
task_parameters.checkpoint_restore_path = args.pretrained_checkpoint_dir
task_parameters.checkpoint_save_dir = args.checkpoint_dir
training_worker(
graph_manager=graph_manager,
task_parameters=task_parameters,
user_batch_size=json.loads(robomaker_hyperparams_json)["batch_size"],
user_episode_per_rollout=json.loads(robomaker_hyperparams_json)["num_episodes_between_training"]
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-p',
'--preset',
help=
"(string) Name of a preset to run (class name from the 'presets' directory.)",
type=str,
required=False)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get("MODEL_S3_BUCKET",
"gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default=os.environ.get("MODEL_S3_PREFIX", "sagemaker"))
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("APP_REGION", "us-east-1"))
parser.add_argument('--number_of_trials',
help='(integer) Number of trials',
type=int,
default=os.environ.get("NUMBER_OF_TRIALS", 15))
parser.add_argument(
'-c',
'--local_model_directory',
help=
'(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--model_metadata_s3_key',
help='(string) Model Metadata File S3 Key',
type=str,
default=os.environ.get("MODEL_METADATA_FILE_S3_KEY",
None))
args = parser.parse_args()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
register(id=defaults.ENV_ID,
entry_point=defaults.ENTRY_POINT,
max_episode_steps=defaults.MAX_STEPS,
reward_threshold=defaults.THRESHOLD)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
# Download the model
# s3_client.download_model(args.local_model_directory)
preset_file_success, _ = download_custom_files_if_present(
s3_client, args.s3_prefix)
if preset_file_success:
preset_location = os.path.join(CUSTOM_FILES_PATH, "preset.py")
preset_location += ":graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
logger.info("Using custom preset file!")
else:
logger.info("Preset file could not be downloaded. Exiting!")
sys.exit(1)
ds_params_instance = S3BotoDataStoreParameters(
bucket_name=args.s3_bucket,
checkpoint_dir=args.local_model_directory,
aws_region=args.aws_region,
s3_folder=args.s3_prefix)
data_store = S3BotoDataStore(ds_params_instance)
data_store.get_a_particular_model(checkpoint_number=1)
graph_manager.data_store = data_store
graph_manager.env_params.seed = 0
evaluation_worker(graph_manager=graph_manager,
number_of_trials=args.number_of_trials,
local_model_directory=args.local_model_directory)
def __init__(self,
level: LevelSelection,
frame_skip: int,
visualization_parameters: VisualizationParameters,
additional_simulator_parameters: Dict[str, Any] = None,
seed: Union[None, int] = None,
human_control: bool = False,
custom_reward_threshold: Union[int, float] = None,
random_initialization_steps: int = 1,
max_over_num_frames: int = 1,
**kwargs):
super().__init__(level, seed, frame_skip, human_control,
custom_reward_threshold, visualization_parameters)
self.random_initialization_steps = random_initialization_steps
self.max_over_num_frames = max_over_num_frames
self.additional_simulator_parameters = additional_simulator_parameters
# hide warnings
gym.logger.set_level(40)
"""
load and initialize environment
environment ids can be defined in 3 ways:
1. Native gym environments like BreakoutDeterministic-v0 for example
2. Custom gym environments written and installed as python packages.
This environments should have a python module with a class inheriting gym.Env, implementing the
relevant functions (_reset, _step, _render) and defining the observation and action space
For example: my_environment_package:MyEnvironmentClass will run an environment defined in the
MyEnvironmentClass class
3. Custom gym environments written as an independent module which is not installed.
This environments should have a python module with a class inheriting gym.Env, implementing the
relevant functions (_reset, _step, _render) and defining the observation and action space.
For example: path_to_my_environment.sub_directory.my_module:MyEnvironmentClass will run an
environment defined in the MyEnvironmentClass class which is located in the module in the relative path
path_to_my_environment.sub_directory.my_module
"""
if ':' in self.env_id:
# custom environments
if '/' in self.env_id or '.' in self.env_id:
# environment in a an absolute path module written as a unix path or in a relative path module
# written as a python import path
env_class = short_dynamic_import(self.env_id)
else:
# environment in a python package
env_class = gym.envs.registration.load(self.env_id)
# instantiate the environment
if self.additional_simulator_parameters:
self.env = env_class(**self.additional_simulator_parameters)
else:
self.env = env_class()
else:
self.env = gym.make(self.env_id)
# for classic control we want to use the native renderer because otherwise we will get 2 renderer windows
environment_to_always_use_with_native_rendering = [
'classic_control', 'mujoco', 'robotics'
]
self.native_rendering = self.native_rendering or \
any([env in str(self.env.unwrapped.__class__)
for env in environment_to_always_use_with_native_rendering])
if self.native_rendering:
if hasattr(self, 'renderer'):
self.renderer.close()
# seed
if self.seed is not None:
self.env.seed(self.seed)
np.random.seed(self.seed)
random.seed(self.seed)
# frame skip and max between consecutive frames
self.is_robotics_env = 'robotics' in str(self.env.unwrapped.__class__)
self.is_mujoco_env = 'mujoco' in str(self.env.unwrapped.__class__)
self.is_atari_env = 'Atari' in str(self.env.unwrapped.__class__)
self.timelimit_env_wrapper = self.env
if self.is_atari_env:
self.env.unwrapped.frameskip = 1 # this accesses the atari env that is wrapped with a timelimit wrapper env
if self.env_id == "SpaceInvadersDeterministic-v4" and self.frame_skip == 4:
screen.warning(
"Warning: The frame-skip for Space Invaders was automatically updated from 4 to 3. "
"This is following the DQN paper where it was noticed that a frame-skip of 3 makes the "
"laser rays disappear. To force frame-skip of 4, please use SpaceInvadersNoFrameskip-v4."
)
self.frame_skip = 3
self.env = MaxOverFramesAndFrameskipEnvWrapper(
self.env,
frameskip=self.frame_skip,
max_over_num_frames=self.max_over_num_frames)
else:
self.env.unwrapped.frameskip = self.frame_skip
self.state_space = StateSpace({})
# observations
if not isinstance(self.env.observation_space,
gym.spaces.dict_space.Dict):
state_space = {'observation': self.env.observation_space}
else:
state_space = self.env.observation_space.spaces
for observation_space_name, observation_space in state_space.items():
if len(observation_space.shape
) == 3 and observation_space.shape[-1] == 3:
# we assume gym has image observations which are RGB and where their values are within 0-255
self.state_space[
observation_space_name] = ImageObservationSpace(
shape=np.array(observation_space.shape),
high=255,
channels_axis=-1)
else:
self.state_space[
observation_space_name] = VectorObservationSpace(
shape=observation_space.shape[0],
low=observation_space.low,
high=observation_space.high)
if 'desired_goal' in state_space.keys():
self.goal_space = self.state_space['desired_goal']
# actions
if type(self.env.action_space) == gym.spaces.box.Box:
self.action_space = BoxActionSpace(
shape=self.env.action_space.shape,
low=self.env.action_space.low,
high=self.env.action_space.high)
elif type(self.env.action_space) == gym.spaces.discrete.Discrete:
actions_description = []
if hasattr(self.env.unwrapped, 'get_action_meanings'):
actions_description = self.env.unwrapped.get_action_meanings()
self.action_space = DiscreteActionSpace(
num_actions=self.env.action_space.n,
descriptions=actions_description)
if self.human_control:
# TODO: add this to the action space
# map keyboard keys to actions
self.key_to_action = {}
if hasattr(self.env.unwrapped, 'get_keys_to_action'):
self.key_to_action = self.env.unwrapped.get_keys_to_action()
# initialize the state by getting a new state from the environment
self.reset_internal_state(True)
# render
if self.is_rendered:
image = self.get_rendered_image()
scale = 1
if self.human_control:
scale = 2
if not self.native_rendering:
self.renderer.create_screen(image.shape[1] * scale,
image.shape[0] * scale)
# measurements
if self.env.spec is not None:
self.timestep_limit = self.env.spec.timestep_limit
else:
self.timestep_limit = None
# the info is only updated after the first step
self.state = self.step(self.action_space.default_action).next_state
self.state_space['measurements'] = VectorObservationSpace(
shape=len(self.info.keys()))
if self.env.spec and custom_reward_threshold is None:
self.reward_success_threshold = self.env.spec.reward_threshold
self.reward_space = RewardSpace(
1, reward_success_threshold=self.reward_success_threshold)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--markov-preset-file',
help="(string) Name of a preset file to run in Markov's preset directory.",
type=str,
default=os.environ.get("MARKOV_PRESET_FILE", "object_tracker.py"))
parser.add_argument('-c', '--local-model-directory',
help='(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default=os.environ.get("LOCAL_MODEL_DIRECTORY", "./checkpoint"))
parser.add_argument('-n', '--num-rollout-workers',
help="(int) Number of workers for multi-process based agents, e.g. A3C",
default=os.environ.get("NUMBER_OF_ROLLOUT_WORKERS", 1),
type=int)
parser.add_argument('--model-s3-bucket',
help='(string) S3 bucket where trained models are stored. It contains model checkpoints.',
type=str,
default=os.environ.get("MODEL_S3_BUCKET"))
parser.add_argument('--model-s3-prefix',
help='(string) S3 prefix where trained models are stored. It contains model checkpoints.',
type=str,
default=os.environ.get("MODEL_S3_PREFIX"))
parser.add_argument('--aws-region',
help='(string) AWS region',
type=str,
default=os.environ.get("ROS_AWS_REGION", "us-west-2"))
args = parser.parse_args()
data_store_params_instance = S3BotoDataStoreParameters(bucket_name=args.model_s3_bucket,
s3_folder=args.model_s3_prefix,
checkpoint_dir=args.local_model_directory,
aws_region=args.aws_region)
data_store = S3BotoDataStore(data_store_params_instance)
# Get the IP of the trainer machine
trainer_ip = data_store.get_ip()
print("Received IP from SageMaker successfully: %s" % trainer_ip)
preset_file_success = data_store.download_presets_if_present(PRESET_LOCAL_PATH)
if preset_file_success:
environment_file_success = data_store.download_environments_if_present(ENVIRONMENT_LOCAL_PATH)
path_and_module = PRESET_LOCAL_PATH + args.markov_preset_file + ":graph_manager"
graph_manager = short_dynamic_import(path_and_module, ignore_module_case=True)
if environment_file_success:
import robomaker.environments
print("Using custom preset file!")
elif args.markov_preset_file:
markov_path = imp.find_module("markov")[1]
preset_location = os.path.join(markov_path, "presets", args.markov_preset_file)
path_and_module = preset_location + ":graph_manager"
graph_manager = short_dynamic_import(path_and_module, ignore_module_case=True)
print("Using custom preset file from Markov presets directory!")
else:
raise ValueError("Unable to determine preset file")
memory_backend_params = RedisPubSubMemoryBackendParameters(redis_address=trainer_ip,
redis_port=TRAINER_REDIS_PORT,
run_type='worker',
channel=args.model_s3_prefix)
graph_manager.agent_params.memory.register_var('memory_backend_params', memory_backend_params)
graph_manager.data_store_params = data_store_params_instance
graph_manager.data_store = data_store
utils.wait_for_checkpoint(checkpoint_dir=args.local_model_directory, data_store=data_store)
rollout_worker(
graph_manager=graph_manager,
checkpoint_dir=args.local_model_directory,
data_store=data_store,
num_workers=args.num_rollout_workers
)
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--checkpoint_dir',
help=
'(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=rospy.get_param("SAGEMAKER_SHARED_S3_BUCKET",
"gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default=rospy.get_param("SAGEMAKER_SHARED_S3_PREFIX",
"sagemaker"))
parser.add_argument(
'--num_workers',
help="(int) The number of workers started in this pool",
type=int,
default=int(rospy.get_param("NUM_WORKERS", 1)))
parser.add_argument('--rollout_idx',
help="(int) The index of current rollout worker",
type=int,
default=0)
parser.add_argument('-r',
'--redis_ip',
help="(string) IP or host for the redis server",
default='localhost',
type=str)
parser.add_argument('-rp',
'--redis_port',
help="(int) Port of the redis server",
default=6379,
type=int)
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=rospy.get_param("AWS_REGION", "us-east-1"))
parser.add_argument('--reward_file_s3_key',
help='(string) Reward File S3 Key',
type=str,
default=rospy.get_param("REWARD_FILE_S3_KEY", None))
parser.add_argument('--model_metadata_s3_key',
help='(string) Model Metadata File S3 Key',
type=str,
default=rospy.get_param("MODEL_METADATA_FILE_S3_KEY",
None))
# For training job, reset is not allowed. penalty_seconds, off_track_penalty, and
# collision_penalty will all be 0 be default
parser.add_argument('--number_of_resets',
help='(integer) Number of resets',
type=int,
default=int(rospy.get_param("NUMBER_OF_RESETS", 0)))
parser.add_argument('--penalty_seconds',
help='(float) penalty second',
type=float,
default=float(rospy.get_param("PENALTY_SECONDS", 0.0)))
parser.add_argument('--job_type',
help='(string) job type',
type=str,
default=rospy.get_param("JOB_TYPE", "TRAINING"))
parser.add_argument('--is_continuous',
help='(boolean) is continous after lap completion',
type=bool,
default=utils.str2bool(
rospy.get_param("IS_CONTINUOUS", False)))
parser.add_argument('--race_type',
help='(string) Race type',
type=str,
default=rospy.get_param("RACE_TYPE", "TIME_TRIAL"))
parser.add_argument('--off_track_penalty',
help='(float) off track penalty second',
type=float,
default=float(rospy.get_param("OFF_TRACK_PENALTY",
0.0)))
parser.add_argument('--collision_penalty',
help='(float) collision penalty second',
type=float,
default=float(rospy.get_param("COLLISION_PENALTY",
0.0)))
args = parser.parse_args()
logger.info("S3 bucket: %s", args.s3_bucket)
logger.info("S3 prefix: %s", args.s3_prefix)
# Download and import reward function
# TODO: replace 'agent' with name of each agent for multi-agent training
reward_function_file = RewardFunction(
bucket=args.s3_bucket,
s3_key=args.reward_file_s3_key,
region_name=args.aws_region,
local_path=REWARD_FUCTION_LOCAL_PATH_FORMAT.format('agent'))
reward_function = reward_function_file.get_reward_function()
# Instantiate Cameras
configure_camera(namespaces=['racecar'])
preset_file_success, _ = download_custom_files_if_present(
s3_bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
# download model metadata
# TODO: replace 'agent' with name of each agent
model_metadata = ModelMetadata(
bucket=args.s3_bucket,
s3_key=args.model_metadata_s3_key,
region_name=args.aws_region,
local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format('agent'))
model_metadata_info = model_metadata.get_model_metadata_info()
version = model_metadata_info[ModelMetadataKeys.VERSION.value]
agent_config = {
'model_metadata': model_metadata,
ConfigParams.CAR_CTRL_CONFIG.value: {
ConfigParams.LINK_NAME_LIST.value:
LINK_NAMES,
ConfigParams.VELOCITY_LIST.value:
VELOCITY_TOPICS,
ConfigParams.STEERING_LIST.value:
STEERING_TOPICS,
ConfigParams.CHANGE_START.value:
utils.str2bool(rospy.get_param('CHANGE_START_POSITION', True)),
ConfigParams.ALT_DIR.value:
utils.str2bool(
rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)),
ConfigParams.MODEL_METADATA.value:
model_metadata,
ConfigParams.REWARD.value:
reward_function,
ConfigParams.AGENT_NAME.value:
'racecar',
ConfigParams.VERSION.value:
version,
ConfigParams.NUMBER_OF_RESETS.value:
args.number_of_resets,
ConfigParams.PENALTY_SECONDS.value:
args.penalty_seconds,
ConfigParams.NUMBER_OF_TRIALS.value:
None,
ConfigParams.IS_CONTINUOUS.value:
args.is_continuous,
ConfigParams.RACE_TYPE.value:
args.race_type,
ConfigParams.COLLISION_PENALTY.value:
args.collision_penalty,
ConfigParams.OFF_TRACK_PENALTY.value:
args.off_track_penalty
}
}
#! TODO each agent should have own s3 bucket
metrics_key = rospy.get_param('METRICS_S3_OBJECT_KEY')
if args.num_workers > 1 and args.rollout_idx > 0:
key_tuple = os.path.splitext(metrics_key)
metrics_key = "{}_{}{}".format(key_tuple[0], str(args.rollout_idx),
key_tuple[1])
metrics_s3_config = {
MetricsS3Keys.METRICS_BUCKET.value:
rospy.get_param('METRICS_S3_BUCKET'),
MetricsS3Keys.METRICS_KEY.value: metrics_key,
MetricsS3Keys.REGION.value: rospy.get_param('AWS_REGION')
}
run_phase_subject = RunPhaseSubject()
agent_list = list()
#TODO: replace agent for multi agent training
# checkpoint s3 instance
# TODO replace agent with agent_0 and so on for multiagent case
checkpoint = Checkpoint(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
region_name=args.aws_region,
agent_name='agent',
checkpoint_dir=args.checkpoint_dir)
agent_list.append(
create_rollout_agent(
agent_config,
TrainingMetrics(
agent_name='agent',
s3_dict_metrics=metrics_s3_config,
deepracer_checkpoint_json=checkpoint.deepracer_checkpoint_json,
ckpnt_dir=os.path.join(args.checkpoint_dir, 'agent'),
run_phase_sink=run_phase_subject,
use_model_picker=(args.rollout_idx == 0)), run_phase_subject))
agent_list.append(create_obstacles_agent())
agent_list.append(create_bot_cars_agent())
# ROS service to indicate all the robomaker markov packages are ready for consumption
signal_robomaker_markov_package_ready()
PhaseObserver('/agent/training_phase', run_phase_subject)
aws_region = rospy.get_param('AWS_REGION', args.aws_region)
simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None)
mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET',
None) if args.rollout_idx == 0 else None
if simtrace_s3_bucket:
simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX')
if args.num_workers > 1:
simtrace_s3_object_prefix = os.path.join(simtrace_s3_object_prefix,
str(args.rollout_idx))
if mp4_s3_bucket:
mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX')
simtrace_video_s3_writers = []
#TODO: replace 'agent' with 'agent_0' for multi agent training and
# mp4_s3_object_prefix, mp4_s3_bucket will be a list, so need to access with index
if simtrace_s3_bucket:
simtrace_video_s3_writers.append(
SimtraceVideo(
upload_type=SimtraceVideoNames.SIMTRACE_TRAINING.value,
bucket=simtrace_s3_bucket,
s3_prefix=simtrace_s3_object_prefix,
region_name=aws_region,
local_path=SIMTRACE_TRAINING_LOCAL_PATH_FORMAT.format(
'agent')))
if mp4_s3_bucket:
simtrace_video_s3_writers.extend([
SimtraceVideo(
upload_type=SimtraceVideoNames.PIP.value,
bucket=mp4_s3_bucket,
s3_prefix=mp4_s3_object_prefix,
region_name=aws_region,
local_path=CAMERA_PIP_MP4_LOCAL_PATH_FORMAT.format('agent')),
SimtraceVideo(
upload_type=SimtraceVideoNames.DEGREE45.value,
bucket=mp4_s3_bucket,
s3_prefix=mp4_s3_object_prefix,
region_name=aws_region,
local_path=CAMERA_45DEGREE_LOCAL_PATH_FORMAT.format('agent')),
SimtraceVideo(
upload_type=SimtraceVideoNames.TOPVIEW.value,
bucket=mp4_s3_bucket,
s3_prefix=mp4_s3_object_prefix,
region_name=aws_region,
local_path=CAMERA_TOPVIEW_LOCAL_PATH_FORMAT.format('agent'))
])
# TODO: replace 'agent' with specific agent name for multi agent training
ip_config = IpConfig(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
region_name=args.aws_region,
local_path=IP_ADDRESS_LOCAL_PATH.format('agent'))
redis_ip = ip_config.get_ip_config()
# Download hyperparameters from SageMaker shared s3 bucket
# TODO: replace 'agent' with name of each agent
hyperparameters = Hyperparameters(
bucket=args.s3_bucket,
s3_key=get_s3_key(args.s3_prefix, HYPERPARAMETER_S3_POSTFIX),
region_name=args.aws_region,
local_path=HYPERPARAMETER_LOCAL_PATH_FORMAT.format('agent'))
sm_hyperparams_dict = hyperparameters.get_hyperparameters_dict()
enable_domain_randomization = utils.str2bool(
rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False))
# Make the clients that will allow us to pause and unpause the physics
rospy.wait_for_service('/gazebo/pause_physics_dr')
rospy.wait_for_service('/gazebo/unpause_physics_dr')
pause_physics = ServiceProxyWrapper('/gazebo/pause_physics_dr', Empty)
unpause_physics = ServiceProxyWrapper('/gazebo/unpause_physics_dr', Empty)
if preset_file_success:
preset_location = os.path.join(CUSTOM_FILES_PATH, "preset.py")
preset_location += ":graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
logger.info("Using custom preset file!")
else:
graph_manager, _ = get_graph_manager(
hp_dict=sm_hyperparams_dict,
agent_list=agent_list,
run_phase_subject=run_phase_subject,
enable_domain_randomization=enable_domain_randomization,
pause_physics=pause_physics,
unpause_physics=unpause_physics)
# If num_episodes_between_training is smaller than num_workers then cancel worker early.
episode_steps_per_rollout = graph_manager.agent_params.algorithm.num_consecutive_playing_steps.num_steps
# Reduce number of workers if allocated more than num_episodes_between_training
if args.num_workers > episode_steps_per_rollout:
logger.info(
"Excess worker allocated. Reducing from {} to {}...".format(
args.num_workers, episode_steps_per_rollout))
args.num_workers = episode_steps_per_rollout
if args.rollout_idx >= episode_steps_per_rollout or args.rollout_idx >= args.num_workers:
err_msg_format = "Exiting excess worker..."
err_msg_format += "(rollout_idx[{}] >= num_workers[{}] or num_episodes_between_training[{}])"
logger.info(
err_msg_format.format(args.rollout_idx, args.num_workers,
episode_steps_per_rollout))
# Close the down the job
utils.cancel_simulation_job()
memory_backend_params = DeepRacerRedisPubSubMemoryBackendParameters(
redis_address=redis_ip,
redis_port=6379,
run_type=str(RunType.ROLLOUT_WORKER),
channel=args.s3_prefix,
num_workers=args.num_workers,
rollout_idx=args.rollout_idx)
graph_manager.memory_backend_params = memory_backend_params
checkpoint_dict = {'agent': checkpoint}
ds_params_instance = S3BotoDataStoreParameters(
checkpoint_dict=checkpoint_dict)
graph_manager.data_store = S3BotoDataStore(ds_params_instance,
graph_manager)
task_parameters = TaskParameters()
task_parameters.checkpoint_restore_path = args.checkpoint_dir
rollout_worker(graph_manager=graph_manager,
num_workers=args.num_workers,
rollout_idx=args.rollout_idx,
task_parameters=task_parameters,
simtrace_video_s3_writers=simtrace_video_s3_writers,
pause_physics=pause_physics,
unpause_physics=unpause_physics)
def main():
screen.set_use_colors(False)
try:
parser = argparse.ArgumentParser()
parser.add_argument(
'-pk',
'--preset_s3_key',
help="(string) Name of a preset to download from S3",
type=str,
required=False)
parser.add_argument(
'-ek',
'--environment_s3_key',
help="(string) Name of an environment file to download from S3",
type=str,
required=False)
parser.add_argument('--model_metadata_s3_key',
help="(string) Model Metadata File S3 Key",
type=str,
required=False)
parser.add_argument(
'-c',
'--checkpoint-dir',
help=
'(string) Path to a folder containing a checkpoint to write the model to.',
type=str,
default='./checkpoint')
parser.add_argument(
'--pretrained-checkpoint-dir',
help=
'(string) Path to a folder for downloading a pre-trained model',
type=str,
default=PRETRAINED_MODEL_DIR)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get(
"SAGEMAKER_SHARED_S3_BUCKET_PATH",
"gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default='sagemaker')
parser.add_argument('--framework',
help='(string) tensorflow or mxnet',
type=str,
default='tensorflow')
parser.add_argument('--pretrained_s3_bucket',
help='(string) S3 bucket for pre-trained model',
type=str)
parser.add_argument('--pretrained_s3_prefix',
help='(string) S3 prefix for pre-trained model',
type=str,
default='sagemaker')
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("AWS_REGION", "us-east-1"))
args, unknown = parser.parse_known_args()
start_redis_server()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
s3_client.upload_file(
os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix),
model_metadata_local_path)
shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR)
# Register the gym enviroment, this will give clients the ability to creat the enviroment object
register(id=defaults.ENV_ID,
entry_point=defaults.ENTRY_POINT,
max_episode_steps=defaults.MAX_STEPS,
reward_threshold=defaults.THRESHOLD)
user_batch_size, user_episode_per_rollout = None, None
success_custom_preset = False
if args.preset_s3_key:
preset_local_path = "./markov/presets/preset.py"
success_custom_preset = s3_client.download_file(
s3_key=args.preset_s3_key, local_path=preset_local_path)
if not success_custom_preset:
logger.info(
"Could not download the preset file. Using the default DeepRacer preset."
)
else:
preset_location = "markov.presets.preset:graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
success_custom_preset = s3_client.upload_file(
s3_key=os.path.normpath("%s/presets/preset.py" %
args.s3_prefix),
local_path=preset_local_path)
if success_custom_preset:
agent_param_loc = "markov.presets.preset:agent_params"
agent_params = short_dynamic_import(
agent_param_loc, ignore_module_case=True)
user_batch_size = agent_params.network_wrappers[
'main'].batch_size
user_episode_per_rollout = agent_params.algorithm.num_consecutive_playing_steps.num_steps
logger.info("Using preset: %s" % args.preset_s3_key)
if not success_custom_preset:
from markov.sagemaker_graph_manager import get_graph_manager
user_batch_size = json.loads(
robomaker_hyperparams_json)["batch_size"],
user_episode_per_rollout = json.loads(
robomaker_hyperparams_json)["num_episodes_between_training"]
params_blob = os.environ.get('SM_TRAINING_ENV', '')
if params_blob:
params = json.loads(params_blob)
sm_hyperparams_dict = params["hyperparameters"]
else:
sm_hyperparams_dict = {}
graph_manager, robomaker_hyperparams_json = get_graph_manager(
**sm_hyperparams_dict)
s3_client.upload_hyperparameters(robomaker_hyperparams_json)
logger.info("Uploaded hyperparameters.json to S3")
host_ip_address = get_ip_from_host()
s3_client.write_ip_config(host_ip_address)
logger.info("Uploaded IP address information to S3: %s" %
host_ip_address)
use_pretrained_model = args.pretrained_s3_bucket and args.pretrained_s3_prefix
if use_pretrained_model:
s3_client_pretrained = SageS3Client(
bucket=args.pretrained_s3_bucket,
s3_prefix=args.pretrained_s3_prefix,
aws_region=args.aws_region)
s3_client_pretrained.download_model(args.pretrained_checkpoint_dir)
memory_backend_params = RedisPubSubMemoryBackendParameters(
redis_address="localhost",
redis_port=6379,
run_type='trainer',
channel=args.s3_prefix)
ds_params_instance = S3BotoDataStoreParameters(
bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
aws_region=args.aws_region,
s3_folder=args.s3_prefix)
graph_manager.data_store_params = ds_params_instance
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
training_worker(graph_manager=graph_manager,
checkpoint_dir=args.checkpoint_dir,
use_pretrained_model=use_pretrained_model,
framework=args.framework,
memory_backend_params=memory_backend_params,
user_batch_size=user_batch_size,
user_episode_per_rollout=user_episode_per_rollout)
except Exception as ex:
utils.json_format_logger(
"Training worker exited with exception: {}".format(ex),
**utils.build_system_error_dict(
utils.SIMAPP_TRAINING_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500))
utils.simapp_exit_gracefully()
def preset_from_name(self, preset_name):
preset_path = self.path_of_main_launcher()
print("Loading preset %s from %s" % (preset_name, preset_path))
preset_path = os.path.join(self.path_of_main_launcher(),preset_name) + '.py:graph_manager'
graph_manager = short_dynamic_import(preset_path, ignore_module_case=True)
return graph_manager
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--checkpoint_dir',
help=
'(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=rospy.get_param("SAGEMAKER_SHARED_S3_BUCKET",
"gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default=rospy.get_param("SAGEMAKER_SHARED_S3_PREFIX",
"sagemaker"))
parser.add_argument(
'--num-workers',
help="(int) The number of workers started in this pool",
type=int,
default=1)
parser.add_argument('-r',
'--redis_ip',
help="(string) IP or host for the redis server",
default='localhost',
type=str)
parser.add_argument('-rp',
'--redis_port',
help="(int) Port of the redis server",
default=6379,
type=int)
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=rospy.get_param("AWS_REGION", "us-east-1"))
parser.add_argument('--reward_file_s3_key',
help='(string) Reward File S3 Key',
type=str,
default=rospy.get_param("REWARD_FILE_S3_KEY", None))
parser.add_argument('--model_metadata_s3_key',
help='(string) Model Metadata File S3 Key',
type=str,
default=rospy.get_param("MODEL_METADATA_FILE_S3_KEY",
None))
args = parser.parse_args()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
logger.info("S3 bucket: %s" % args.s3_bucket)
logger.info("S3 prefix: %s" % args.s3_prefix)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
utils.load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
# Download and import reward function
if not args.reward_file_s3_key:
utils.log_and_exit(
"Reward function code S3 key not available for S3 bucket {} and prefix {}"
.format(args.s3_bucket,
args.s3_prefix), utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500)
download_customer_reward_function(s3_client, args.reward_file_s3_key)
try:
from custom_files.customer_reward_function import reward_function
except Exception as e:
utils.log_and_exit(
"Failed to import user's reward_function: {}".format(e),
utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_400)
# Instantiate Cameras
configure_camera()
redis_ip = s3_client.get_ip()
logger.info("Received IP from SageMaker successfully: %s" % redis_ip)
# Download hyperparameters from SageMaker
hyperparameters_file_success = False
hyperparams_s3_key = os.path.normpath(args.s3_prefix +
"/ip/hyperparameters.json")
hyperparameters_file_success = s3_client.download_file(
s3_key=hyperparams_s3_key, local_path="hyperparameters.json")
sm_hyperparams_dict = {}
if hyperparameters_file_success:
logger.info("Received Sagemaker hyperparameters successfully!")
with open("hyperparameters.json") as fp:
sm_hyperparams_dict = json.load(fp)
else:
logger.info("SageMaker hyperparameters not found.")
preset_file_success, _ = download_custom_files_if_present(
s3_client, args.s3_prefix)
#! TODO each agent should have own config
_, _, version = utils_parse_model_metadata.parse_model_metadata(
model_metadata_local_path)
agent_config = {
'model_metadata': model_metadata_local_path,
'car_ctrl_cnfig': {
ConfigParams.LINK_NAME_LIST.value:
LINK_NAMES,
ConfigParams.VELOCITY_LIST.value:
VELOCITY_TOPICS,
ConfigParams.STEERING_LIST.value:
STEERING_TOPICS,
ConfigParams.CHANGE_START.value:
utils.str2bool(rospy.get_param('CHANGE_START_POSITION', True)),
ConfigParams.ALT_DIR.value:
utils.str2bool(
rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)),
ConfigParams.ACTION_SPACE_PATH.value:
'custom_files/model_metadata.json',
ConfigParams.REWARD.value:
reward_function,
ConfigParams.AGENT_NAME.value:
'racecar',
ConfigParams.VERSION.value:
version
}
}
#! TODO each agent should have own s3 bucket
metrics_s3_config = {
MetricsS3Keys.METRICS_BUCKET.value:
rospy.get_param('METRICS_S3_BUCKET'),
MetricsS3Keys.METRICS_KEY.value:
rospy.get_param('METRICS_S3_OBJECT_KEY'),
MetricsS3Keys.REGION.value:
rospy.get_param('AWS_REGION'),
MetricsS3Keys.STEP_BUCKET.value:
rospy.get_param('SAGEMAKER_SHARED_S3_BUCKET'),
MetricsS3Keys.STEP_KEY.value:
os.path.join(rospy.get_param('SAGEMAKER_SHARED_S3_PREFIX'),
TRAINING_SIMTRACE_DATA_S3_OBJECT_KEY)
}
agent_list = list()
agent_list.append(
create_rollout_agent(agent_config, TrainingMetrics(metrics_s3_config)))
agent_list.append(create_obstacles_agent())
agent_list.append(create_bot_cars_agent())
if preset_file_success:
preset_location = os.path.join(CUSTOM_FILES_PATH, "preset.py")
preset_location += ":graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
logger.info("Using custom preset file!")
else:
graph_manager, _ = get_graph_manager(sm_hyperparams_dict, agent_list)
memory_backend_params = RedisPubSubMemoryBackendParameters(
redis_address=redis_ip,
redis_port=6379,
run_type=str(RunType.ROLLOUT_WORKER),
channel=args.s3_prefix)
graph_manager.memory_backend_params = memory_backend_params
ds_params_instance = S3BotoDataStoreParameters(
aws_region=args.aws_region,
bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
s3_folder=args.s3_prefix)
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
task_parameters = TaskParameters()
task_parameters.checkpoint_restore_path = args.checkpoint_dir
rollout_worker(graph_manager=graph_manager,
data_store=data_store,
num_workers=args.num_workers,
task_parameters=task_parameters)
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--checkpoint-dir',
help='(string) Path to a local folder containing a checkpoint to write the model to.',
type=str,
default='./checkpoint')
parser.add_argument('--pretrained-checkpoint-dir',
help='(string) Path to a local folder for downloading a pre-trained model',
type=str,
default=PRETRAINED_MODEL_DIR)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get("SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default='sagemaker')
parser.add_argument('--framework',
help='(string) tensorflow or mxnet',
type=str,
default='tensorflow')
parser.add_argument('--pretrained_s3_bucket',
help='(string) S3 bucket for pre-trained model',
type=str)
parser.add_argument('--pretrained_s3_prefix',
help='(string) S3 prefix for pre-trained model',
type=str,
default='sagemaker')
parser.add_argument('--RLCOACH_PRESET',
help='(string) Default preset to use',
type=str,
default='deepracer')
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
required=True)
args, unknown = parser.parse_known_args()
s3_client = SageS3Client(bucket=args.s3_bucket, s3_prefix=args.s3_prefix, aws_region=args.aws_region)
# Import to register the environment with Gym
import robomaker.environments
preset_location = "robomaker.presets.%s:graph_manager" % args.RLCOACH_PRESET
graph_manager = short_dynamic_import(preset_location, ignore_module_case=True)
host_ip_address = get_ip_from_host()
s3_client.write_ip_config(host_ip_address)
print("Uploaded IP address information to S3: %s" % host_ip_address)
use_pretrained_model = False
if args.pretrained_s3_bucket and args.pretrained_s3_prefix:
s3_client_pretrained = SageS3Client(bucket=args.pretrained_s3_bucket,
s3_prefix=args.pretrained_s3_prefix,
aws_region=args.aws_region)
s3_client_pretrained.download_model(PRETRAINED_MODEL_DIR)
use_pretrained_model = True
memory_backend_params = RedisPubSubMemoryBackendParameters(redis_address="localhost",
redis_port=6379,
run_type='trainer',
channel=args.s3_prefix)
graph_manager.agent_params.memory.register_var('memory_backend_params', memory_backend_params)
ds_params_instance = S3BotoDataStoreParameters(bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
s3_folder=args.s3_prefix,
aws_region=args.aws_region)
graph_manager.data_store_params = ds_params_instance
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
training_worker(
graph_manager=graph_manager,
checkpoint_dir=args.checkpoint_dir,
use_pretrained_model=use_pretrained_model,
framework=args.framework
)
def parse_arguments(parser: argparse.ArgumentParser) -> argparse.Namespace:
"""
Parse the arguments that the user entered
:param parser: the argparse command line parser
:return: the parsed arguments
"""
args = parser.parse_args()
# if no arg is given
if len(sys.argv) == 1:
parser.print_help()
exit(0)
# list available presets
preset_names = list_all_presets()
if args.list:
screen.log_title("Available Presets:")
for preset in sorted(preset_names):
print(preset)
sys.exit(0)
# replace a short preset name with the full path
if args.preset is not None:
if args.preset.lower() in [p.lower() for p in preset_names]:
args.preset = "{}.py:graph_manager".format(
os.path.join(get_base_dir(), 'presets', args.preset))
else:
args.preset = "{}".format(args.preset)
# if a graph manager variable was not specified, try the default of :graph_manager
if len(args.preset.split(":")) == 1:
args.preset += ":graph_manager"
# verify that the preset exists
preset_path = args.preset.split(":")[0]
if not os.path.exists(preset_path):
screen.error("The given preset ({}) cannot be found.".format(
args.preset))
# verify that the preset can be instantiated
try:
short_dynamic_import(args.preset, ignore_module_case=True)
except TypeError as e:
traceback.print_exc()
screen.error('Internal Error: ' + str(e) +
"\n\nThe given preset ({}) cannot be instantiated.".
format(args.preset))
# validate the checkpoints args
if args.checkpoint_restore_dir is not None and not os.path.exists(
args.checkpoint_restore_dir):
screen.error(
"The requested checkpoint folder to load from does not exist.")
# no preset was given. check if the user requested to play some environment on its own
if args.preset is None and args.play:
if args.environment_type:
args.agent_type = 'Human'
else:
screen.error(
'When no preset is given for Coach to run, and the user requests human control over '
'the environment, the user is expected to input the desired environment_type and level.'
'\nAt least one of these parameters was not given.')
elif args.preset and args.play:
screen.error(
"Both the --preset and the --play flags were set. These flags can not be used together. "
"For human control, please use the --play flag together with the environment type flag (-et)"
)
elif args.preset is None and not args.play:
screen.error(
"Please choose a preset using the -p flag or use the --play flag together with choosing an "
"environment type (-et) in order to play the game.")
# get experiment name and path
args.experiment_name = logger.get_experiment_name(args.experiment_name)
args.experiment_path = logger.get_experiment_path(args.experiment_name)
if args.play and args.num_workers > 1:
screen.warning(
"Playing the game as a human is only available with a single worker. "
"The number of workers will be reduced to 1")
args.num_workers = 1
args.framework = Frameworks[args.framework.lower()]
# checkpoints
args.save_checkpoint_dir = os.path.join(
args.experiment_path,
'checkpoint') if args.save_checkpoint_secs is not None else None
return args
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--checkpoint_dir',
help=
'(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=rospy.get_param("SAGEMAKER_SHARED_S3_BUCKET",
"gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default=rospy.get_param("SAGEMAKER_SHARED_S3_PREFIX",
"sagemaker"))
parser.add_argument(
'--num_workers',
help="(int) The number of workers started in this pool",
type=int,
default=int(rospy.get_param("NUM_WORKERS", 1)))
parser.add_argument('--rollout_idx',
help="(int) The index of current rollout worker",
type=int,
default=0)
parser.add_argument('-r',
'--redis_ip',
help="(string) IP or host for the redis server",
default='localhost',
type=str)
parser.add_argument('-rp',
'--redis_port',
help="(int) Port of the redis server",
default=6379,
type=int)
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=rospy.get_param("AWS_REGION", "us-east-1"))
parser.add_argument('--reward_file_s3_key',
help='(string) Reward File S3 Key',
type=str,
default=rospy.get_param("REWARD_FILE_S3_KEY", None))
parser.add_argument('--model_metadata_s3_key',
help='(string) Model Metadata File S3 Key',
type=str,
default=rospy.get_param("MODEL_METADATA_FILE_S3_KEY",
None))
# For training job, reset is not allowed. penalty_seconds, off_track_penalty, and
# collision_penalty will all be 0 be default
parser.add_argument('--number_of_resets',
help='(integer) Number of resets',
type=int,
default=int(rospy.get_param("NUMBER_OF_RESETS", 0)))
parser.add_argument('--penalty_seconds',
help='(float) penalty second',
type=float,
default=float(rospy.get_param("PENALTY_SECONDS", 0.0)))
parser.add_argument('--job_type',
help='(string) job type',
type=str,
default=rospy.get_param("JOB_TYPE", "TRAINING"))
parser.add_argument('--is_continuous',
help='(boolean) is continous after lap completion',
type=bool,
default=utils.str2bool(
rospy.get_param("IS_CONTINUOUS", False)))
parser.add_argument('--race_type',
help='(string) Race type',
type=str,
default=rospy.get_param("RACE_TYPE", "TIME_TRIAL"))
parser.add_argument('--off_track_penalty',
help='(float) off track penalty second',
type=float,
default=float(rospy.get_param("OFF_TRACK_PENALTY",
0.0)))
parser.add_argument('--collision_penalty',
help='(float) collision penalty second',
type=float,
default=float(rospy.get_param("COLLISION_PENALTY",
0.0)))
args = parser.parse_args()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
logger.info("S3 bucket: %s", args.s3_bucket)
logger.info("S3 prefix: %s", args.s3_prefix)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
utils.load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
# Download and import reward function
if not args.reward_file_s3_key:
log_and_exit(
"Reward function code S3 key not available for S3 bucket {} and prefix {}"
.format(args.s3_bucket, args.s3_prefix),
SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
download_customer_reward_function(s3_client, args.reward_file_s3_key)
try:
from custom_files.customer_reward_function import reward_function
except Exception as e:
log_and_exit("Failed to import user's reward_function: {}".format(e),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
# Instantiate Cameras
configure_camera(namespaces=['racecar'])
preset_file_success, _ = download_custom_files_if_present(
s3_client, args.s3_prefix)
#! TODO each agent should have own config
_, _, version = utils_parse_model_metadata.parse_model_metadata(
model_metadata_local_path)
agent_config = {
'model_metadata': model_metadata_local_path,
ConfigParams.CAR_CTRL_CONFIG.value: {
ConfigParams.LINK_NAME_LIST.value:
LINK_NAMES,
ConfigParams.VELOCITY_LIST.value:
VELOCITY_TOPICS,
ConfigParams.STEERING_LIST.value:
STEERING_TOPICS,
ConfigParams.CHANGE_START.value:
utils.str2bool(rospy.get_param('CHANGE_START_POSITION', True)),
ConfigParams.ALT_DIR.value:
utils.str2bool(
rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)),
ConfigParams.ACTION_SPACE_PATH.value:
'custom_files/model_metadata.json',
ConfigParams.REWARD.value:
reward_function,
ConfigParams.AGENT_NAME.value:
'racecar',
ConfigParams.VERSION.value:
version,
ConfigParams.NUMBER_OF_RESETS.value:
args.number_of_resets,
ConfigParams.PENALTY_SECONDS.value:
args.penalty_seconds,
ConfigParams.NUMBER_OF_TRIALS.value:
None,
ConfigParams.IS_CONTINUOUS.value:
args.is_continuous,
ConfigParams.RACE_TYPE.value:
args.race_type,
ConfigParams.COLLISION_PENALTY.value:
args.collision_penalty,
ConfigParams.OFF_TRACK_PENALTY.value:
args.off_track_penalty
}
}
#! TODO each agent should have own s3 bucket
step_metrics_prefix = rospy.get_param('SAGEMAKER_SHARED_S3_PREFIX')
if args.num_workers > 1:
step_metrics_prefix = os.path.join(step_metrics_prefix,
str(args.rollout_idx))
metrics_s3_config = {
MetricsS3Keys.METRICS_BUCKET.value:
rospy.get_param('METRICS_S3_BUCKET'),
MetricsS3Keys.METRICS_KEY.value:
rospy.get_param('METRICS_S3_OBJECT_KEY'),
MetricsS3Keys.REGION.value: rospy.get_param('AWS_REGION')
}
metrics_s3_model_cfg = {
MetricsS3Keys.METRICS_BUCKET.value:
args.s3_bucket,
MetricsS3Keys.METRICS_KEY.value:
os.path.join(args.s3_prefix, DEEPRACER_CHKPNT_KEY_SUFFIX),
MetricsS3Keys.REGION.value:
args.aws_region
}
run_phase_subject = RunPhaseSubject()
agent_list = list()
agent_list.append(
create_rollout_agent(
agent_config,
TrainingMetrics(agent_name='agent',
s3_dict_metrics=metrics_s3_config,
s3_dict_model=metrics_s3_model_cfg,
ckpnt_dir=args.checkpoint_dir,
run_phase_sink=run_phase_subject,
use_model_picker=(args.rollout_idx == 0)),
run_phase_subject))
agent_list.append(create_obstacles_agent())
agent_list.append(create_bot_cars_agent())
# ROS service to indicate all the robomaker markov packages are ready for consumption
signal_robomaker_markov_package_ready()
PhaseObserver('/agent/training_phase', run_phase_subject)
aws_region = rospy.get_param('AWS_REGION', args.aws_region)
simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None)
mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET',
None) if args.rollout_idx == 0 else None
if simtrace_s3_bucket:
simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX')
if args.num_workers > 1:
simtrace_s3_object_prefix = os.path.join(simtrace_s3_object_prefix,
str(args.rollout_idx))
if mp4_s3_bucket:
mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX')
s3_writer_job_info = []
if simtrace_s3_bucket:
s3_writer_job_info.append(
IterationData(
'simtrace', simtrace_s3_bucket, simtrace_s3_object_prefix,
aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, 'agent',
IterationDataLocalFileNames.SIM_TRACE_TRAINING_LOCAL_FILE.
value)))
if mp4_s3_bucket:
s3_writer_job_info.extend([
IterationData(
'pip', mp4_s3_bucket, mp4_s3_object_prefix, aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, 'agent',
IterationDataLocalFileNames.
CAMERA_PIP_MP4_VALIDATION_LOCAL_PATH.value)),
IterationData(
'45degree', mp4_s3_bucket, mp4_s3_object_prefix, aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, 'agent',
IterationDataLocalFileNames.
CAMERA_45DEGREE_MP4_VALIDATION_LOCAL_PATH.value)),
IterationData(
'topview', mp4_s3_bucket, mp4_s3_object_prefix, aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, 'agent',
IterationDataLocalFileNames.
CAMERA_TOPVIEW_MP4_VALIDATION_LOCAL_PATH.value))
])
s3_writer = S3Writer(job_info=s3_writer_job_info)
redis_ip = s3_client.get_ip()
logger.info("Received IP from SageMaker successfully: %s", redis_ip)
# Download hyperparameters from SageMaker
hyperparameters_file_success = False
hyperparams_s3_key = os.path.normpath(args.s3_prefix +
"/ip/hyperparameters.json")
hyperparameters_file_success = s3_client.download_file(
s3_key=hyperparams_s3_key, local_path="hyperparameters.json")
sm_hyperparams_dict = {}
if hyperparameters_file_success:
logger.info("Received Sagemaker hyperparameters successfully!")
with open("hyperparameters.json") as filepointer:
sm_hyperparams_dict = json.load(filepointer)
else:
logger.info("SageMaker hyperparameters not found.")
enable_domain_randomization = utils.str2bool(
rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False))
if preset_file_success:
preset_location = os.path.join(CUSTOM_FILES_PATH, "preset.py")
preset_location += ":graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
logger.info("Using custom preset file!")
else:
graph_manager, _ = get_graph_manager(
hp_dict=sm_hyperparams_dict,
agent_list=agent_list,
run_phase_subject=run_phase_subject,
enable_domain_randomization=enable_domain_randomization)
# If num_episodes_between_training is smaller than num_workers then cancel worker early.
episode_steps_per_rollout = graph_manager.agent_params.algorithm.num_consecutive_playing_steps.num_steps
# Reduce number of workers if allocated more than num_episodes_between_training
if args.num_workers > episode_steps_per_rollout:
logger.info(
"Excess worker allocated. Reducing from {} to {}...".format(
args.num_workers, episode_steps_per_rollout))
args.num_workers = episode_steps_per_rollout
if args.rollout_idx >= episode_steps_per_rollout or args.rollout_idx >= args.num_workers:
err_msg_format = "Exiting excess worker..."
err_msg_format += "(rollout_idx[{}] >= num_workers[{}] or num_episodes_between_training[{}])"
logger.info(
err_msg_format.format(args.rollout_idx, args.num_workers,
episode_steps_per_rollout))
# Close the down the job
utils.cancel_simulation_job(
os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'),
rospy.get_param('AWS_REGION'))
memory_backend_params = DeepRacerRedisPubSubMemoryBackendParameters(
redis_address=redis_ip,
redis_port=6379,
run_type=str(RunType.ROLLOUT_WORKER),
channel=args.s3_prefix,
num_workers=args.num_workers,
rollout_idx=args.rollout_idx)
graph_manager.memory_backend_params = memory_backend_params
ds_params_instance = S3BotoDataStoreParameters(
aws_region=args.aws_region,
bucket_names={'agent': args.s3_bucket},
base_checkpoint_dir=args.checkpoint_dir,
s3_folders={'agent': args.s3_prefix})
graph_manager.data_store = S3BotoDataStore(ds_params_instance,
graph_manager)
task_parameters = TaskParameters()
task_parameters.checkpoint_restore_path = args.checkpoint_dir
rollout_worker(graph_manager=graph_manager,
num_workers=args.num_workers,
rollout_idx=args.rollout_idx,
task_parameters=task_parameters,
s3_writer=s3_writer)
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument('-pk',
'--preset_s3_key',
help="(string) Name of a preset to download from S3",
type=str,
required=False)
parser.add_argument(
'-ek',
'--environment_s3_key',
help="(string) Name of an environment file to download from S3",
type=str,
required=False)
parser.add_argument('--model_metadata_s3_key',
help="(string) Model Metadata File S3 Key",
type=str,
required=False)
parser.add_argument(
'-c',
'--checkpoint_dir',
help=
'(string) Path to a folder containing a checkpoint to write the model to.',
type=str,
default='./checkpoint')
parser.add_argument(
'--pretrained_checkpoint_dir',
help='(string) Path to a folder for downloading a pre-trained model',
type=str,
default=PRETRAINED_MODEL_DIR)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get(
"SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default='sagemaker')
parser.add_argument('--framework',
help='(string) tensorflow or mxnet',
type=str,
default='tensorflow')
parser.add_argument('--pretrained_s3_bucket',
help='(string) S3 bucket for pre-trained model',
type=str)
parser.add_argument('--pretrained_s3_prefix',
help='(string) S3 prefix for pre-trained model',
type=str,
default='sagemaker')
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("AWS_REGION", "us-east-1"))
args, _ = parser.parse_known_args()
s3_client = S3Client(region_name=args.aws_region, max_retry_attempts=0)
# download model metadata
# TODO: replace 'agent' with name of each agent
model_metadata_download = ModelMetadata(
bucket=args.s3_bucket,
s3_key=args.model_metadata_s3_key,
region_name=args.aws_region,
local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format('agent'))
model_metadata_info = model_metadata_download.get_model_metadata_info()
network_type = model_metadata_info[ModelMetadataKeys.NEURAL_NETWORK.value]
version = model_metadata_info[ModelMetadataKeys.VERSION.value]
# upload model metadata
model_metadata_upload = ModelMetadata(
bucket=args.s3_bucket,
s3_key=get_s3_key(args.s3_prefix, MODEL_METADATA_S3_POSTFIX),
region_name=args.aws_region,
local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format('agent'))
model_metadata_upload.persist(
s3_kms_extra_args=utils.get_s3_kms_extra_args())
shutil.copy2(model_metadata_download.local_path, SM_MODEL_OUTPUT_DIR)
success_custom_preset = False
if args.preset_s3_key:
preset_local_path = "./markov/presets/preset.py"
try:
s3_client.download_file(bucket=args.s3_bucket,
s3_key=args.preset_s3_key,
local_path=preset_local_path)
success_custom_preset = True
except botocore.exceptions.ClientError:
pass
if not success_custom_preset:
logger.info(
"Could not download the preset file. Using the default DeepRacer preset."
)
else:
preset_location = "markov.presets.preset:graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
s3_client.upload_file(
bucket=args.s3_bucket,
s3_key=os.path.normpath("%s/presets/preset.py" %
args.s3_prefix),
local_path=preset_local_path,
s3_kms_extra_args=utils.get_s3_kms_extra_args())
if success_custom_preset:
logger.info("Using preset: %s" % args.preset_s3_key)
if not success_custom_preset:
params_blob = os.environ.get('SM_TRAINING_ENV', '')
if params_blob:
params = json.loads(params_blob)
sm_hyperparams_dict = params["hyperparameters"]
else:
sm_hyperparams_dict = {}
#! TODO each agent should have own config
agent_config = {
'model_metadata': model_metadata_download,
ConfigParams.CAR_CTRL_CONFIG.value: {
ConfigParams.LINK_NAME_LIST.value: [],
ConfigParams.VELOCITY_LIST.value: {},
ConfigParams.STEERING_LIST.value: {},
ConfigParams.CHANGE_START.value: None,
ConfigParams.ALT_DIR.value: None,
ConfigParams.MODEL_METADATA.value: model_metadata_download,
ConfigParams.REWARD.value: None,
ConfigParams.AGENT_NAME.value: 'racecar'
}
}
agent_list = list()
agent_list.append(create_training_agent(agent_config))
graph_manager, robomaker_hyperparams_json = get_graph_manager(
hp_dict=sm_hyperparams_dict,
agent_list=agent_list,
run_phase_subject=None,
run_type=str(RunType.TRAINER))
# Upload hyperparameters to SageMaker shared s3 bucket
hyperparameters = Hyperparameters(bucket=args.s3_bucket,
s3_key=get_s3_key(
args.s3_prefix,
HYPERPARAMETER_S3_POSTFIX),
region_name=args.aws_region)
hyperparameters.persist(
hyperparams_json=robomaker_hyperparams_json,
s3_kms_extra_args=utils.get_s3_kms_extra_args())
# Attach sample collector to graph_manager only if sample count > 0
max_sample_count = int(sm_hyperparams_dict.get("max_sample_count", 0))
if max_sample_count > 0:
sample_collector = SampleCollector(
bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
region_name=args.aws_region,
max_sample_count=max_sample_count,
sampling_frequency=int(
sm_hyperparams_dict.get("sampling_frequency", 1)))
graph_manager.sample_collector = sample_collector
# persist IP config from sagemaker to s3
ip_config = IpConfig(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
region_name=args.aws_region)
ip_config.persist(s3_kms_extra_args=utils.get_s3_kms_extra_args())
training_algorithm = model_metadata_download.training_algorithm
output_head_format = FROZEN_HEAD_OUTPUT_GRAPH_FORMAT_MAPPING[
training_algorithm]
use_pretrained_model = args.pretrained_s3_bucket and args.pretrained_s3_prefix
# Handle backward compatibility
if use_pretrained_model:
# checkpoint s3 instance for pretrained model
# TODO: replace 'agent' for multiagent training
checkpoint = Checkpoint(bucket=args.pretrained_s3_bucket,
s3_prefix=args.pretrained_s3_prefix,
region_name=args.aws_region,
agent_name='agent',
checkpoint_dir=args.pretrained_checkpoint_dir,
output_head_format=output_head_format)
# make coach checkpoint compatible
if version < SIMAPP_VERSION_2 and not checkpoint.rl_coach_checkpoint.is_compatible(
):
checkpoint.rl_coach_checkpoint.make_compatible(
checkpoint.syncfile_ready)
# get best model checkpoint string
model_checkpoint_name = checkpoint.deepracer_checkpoint_json.get_deepracer_best_checkpoint(
)
# Select the best checkpoint model by uploading rl coach .coach_checkpoint file
checkpoint.rl_coach_checkpoint.update(
model_checkpoint_name=model_checkpoint_name,
s3_kms_extra_args=utils.get_s3_kms_extra_args())
# add checkpoint into checkpoint_dict
checkpoint_dict = {'agent': checkpoint}
# load pretrained model
ds_params_instance_pretrained = S3BotoDataStoreParameters(
checkpoint_dict=checkpoint_dict)
data_store_pretrained = S3BotoDataStore(ds_params_instance_pretrained,
graph_manager, True)
data_store_pretrained.load_from_store()
memory_backend_params = DeepRacerRedisPubSubMemoryBackendParameters(
redis_address="localhost",
redis_port=6379,
run_type=str(RunType.TRAINER),
channel=args.s3_prefix,
network_type=network_type)
graph_manager.memory_backend_params = memory_backend_params
# checkpoint s3 instance for training model
checkpoint = Checkpoint(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
region_name=args.aws_region,
agent_name='agent',
checkpoint_dir=args.checkpoint_dir,
output_head_format=output_head_format)
checkpoint_dict = {'agent': checkpoint}
ds_params_instance = S3BotoDataStoreParameters(
checkpoint_dict=checkpoint_dict)
graph_manager.data_store_params = ds_params_instance
graph_manager.data_store = S3BotoDataStore(ds_params_instance,
graph_manager)
task_parameters = TaskParameters()
task_parameters.experiment_path = SM_MODEL_OUTPUT_DIR
task_parameters.checkpoint_save_secs = 20
if use_pretrained_model:
task_parameters.checkpoint_restore_path = args.pretrained_checkpoint_dir
task_parameters.checkpoint_save_dir = args.checkpoint_dir
training_worker(
graph_manager=graph_manager,
task_parameters=task_parameters,
user_batch_size=json.loads(robomaker_hyperparams_json)["batch_size"],
user_episode_per_rollout=json.loads(
robomaker_hyperparams_json)["num_episodes_between_training"],
training_algorithm=training_algorithm)
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--checkpoint-dir',
help=
'(string) Path to a local folder containing a checkpoint to write the model to.',
type=str,
default='./checkpoint')
parser.add_argument(
'--pretrained-checkpoint-dir',
help=
'(string) Path to a local folder for downloading a pre-trained model',
type=str,
default=PRETRAINED_MODEL_DIR)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get(
"SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default='sagemaker')
parser.add_argument('--framework',
help='(string) tensorflow or mxnet',
type=str,
default='tensorflow')
parser.add_argument('--pretrained_s3_bucket',
help='(string) S3 bucket for pre-trained model',
type=str)
parser.add_argument('--pretrained_s3_prefix',
help='(string) S3 prefix for pre-trained model',
type=str,
default='sagemaker')
parser.add_argument('--RLCOACH_PRESET',
help='(string) Default preset to use',
type=str,
default='object_tracker')
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
required=True)
args, unknown = parser.parse_known_args()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
# Import to register the environment with Gym
import robomaker.environments
preset_location = "robomaker.presets.%s:graph_manager" % args.RLCOACH_PRESET
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
host_ip_address = get_ip_from_host()
s3_client.write_ip_config(host_ip_address)
print("Uploaded IP address information to S3: %s" % host_ip_address)
use_pretrained_model = False
if args.pretrained_s3_bucket and args.pretrained_s3_prefix:
s3_client_pretrained = SageS3Client(
bucket=args.pretrained_s3_bucket,
s3_prefix=args.pretrained_s3_prefix,
aws_region=args.aws_region)
s3_client_pretrained.download_model(PRETRAINED_MODEL_DIR)
use_pretrained_model = True
memory_backend_params = RedisPubSubMemoryBackendParameters(
redis_address="localhost",
redis_port=6379,
run_type='trainer',
channel=args.s3_prefix)
graph_manager.agent_params.memory.register_var('memory_backend_params',
memory_backend_params)
ds_params_instance = S3BotoDataStoreParameters(
bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
s3_folder=args.s3_prefix,
aws_region=args.aws_region)
graph_manager.data_store_params = ds_params_instance
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
training_worker(graph_manager=graph_manager,
checkpoint_dir=args.checkpoint_dir,
use_pretrained_model=use_pretrained_model,
framework=args.framework)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--markov-preset-file',
help="(string) Name of a preset file to run in Markov's preset directory.",
type=str,
default=os.environ.get("MARKOV_PRESET_FILE", "mars_presets.py"))
parser.add_argument('-c', '--local_model_directory',
help='(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default=os.environ.get("LOCAL_MODEL_DIRECTORY", "./checkpoint"))
parser.add_argument('-n', '--num_workers',
help="(int) Number of workers for multi-process based agents, e.g. A3C",
default=1,
type=int)
parser.add_argument('--model-s3-bucket',
help='(string) S3 bucket where trained models are stored. It contains model checkpoints.',
type=str,
default=os.environ.get("MODEL_S3_BUCKET"))
parser.add_argument('--model-s3-prefix',
help='(string) S3 prefix where trained models are stored. It contains model checkpoints.',
type=str,
default=os.environ.get("MODEL_S3_PREFIX"))
parser.add_argument('--aws-region',
help='(string) AWS region',
type=str,
default=os.environ.get("ROS_AWS_REGION", "us-west-1"))
parser.add_argument('--checkpoint-save-secs',
help="(int) Time period in second between 2 checkpoints",
type=int,
default=600)
parser.add_argument('--save-frozen-graph',
help="(bool) True if we need to store the frozen graph",
type=bool,
default=True)
args = parser.parse_args()
if args.markov_preset_file:
markov_path = imp.find_module("markov")[1]
preset_location = os.path.join(markov_path, "presets", args.markov_preset_file)
path_and_module = preset_location + ":graph_manager"
graph_manager = short_dynamic_import(path_and_module, ignore_module_case=True)
else:
raise ValueError("Unable to determine preset file")
# TODO: support other frameworks
task_parameters = TaskParameters(framework_type=Frameworks.tensorflow,
checkpoint_save_secs=args.checkpoint_save_secs)
task_parameters.__dict__['checkpoint_save_dir'] = args.local_model_directory
task_parameters.__dict__ = add_items_to_dict(task_parameters.__dict__, args.__dict__)
# data_store_params_instance = S3BotoDataStoreParameters(bucket_name=args.model_s3_bucket,
# s3_folder=args.model_s3_prefix,
# checkpoint_dir=args.local_model_directory,
# aws_region=args.aws_region)
#data_store = S3BotoDataStore(data_store_params_instance)
#if args.save_frozen_graph:
# data_store.graph_manager = graph_manager
#graph_manager.data_store_params = data_store_params_instance
#graph_manager.data_store = data_store
graph_manager.should_stop = should_stop_training_based_on_evaluation
start_graph(graph_manager=graph_manager, task_parameters=task_parameters)
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--checkpoint_dir',
help=
'(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get("SAGEMAKER_SHARED_S3_BUCKET",
"gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default=os.environ.get("SAGEMAKER_SHARED_S3_PREFIX",
"sagemaker"))
parser.add_argument(
'--num-workers',
help="(int) The number of workers started in this pool",
type=int,
default=1)
parser.add_argument('-r',
'--redis_ip',
help="(string) IP or host for the redis server",
default='localhost',
type=str)
parser.add_argument('-rp',
'--redis_port',
help="(int) Port of the redis server",
default=6379,
type=int)
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("APP_REGION", "us-east-1"))
parser.add_argument('--reward_file_s3_key',
help='(string) Reward File S3 Key',
type=str,
default=os.environ.get("REWARD_FILE_S3_KEY", None))
parser.add_argument('--model_metadata_s3_key',
help='(string) Model Metadata File S3 Key',
type=str,
default=os.environ.get("MODEL_METADATA_FILE_S3_KEY",
None))
parser.add_argument('--aws_endpoint_url',
help='(string) AWS region',
type=str,
default=os.environ.get("AWS_ENDPOINT_URL", None))
args = parser.parse_args()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region,
endpoint_url=args.aws_endpoint_url)
logger.info("S3 bucket: %s" % args.s3_bucket)
logger.info("S3 prefix: %s" % args.s3_prefix)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
# Download reward function
if not args.reward_file_s3_key:
utils.json_format_logger(
"Reward function code S3 key not available for S3 bucket {} and prefix {}"
.format(args.s3_bucket, args.s3_prefix),
**utils.build_system_error_dict(
utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500))
traceback.print_exc()
utils.simapp_exit_gracefully()
download_customer_reward_function(s3_client, args.reward_file_s3_key)
# Register the gym enviroment, this will give clients the ability to creat the enviroment object
register(id=defaults.ENV_ID,
entry_point=defaults.ENTRY_POINT,
max_episode_steps=defaults.MAX_STEPS,
reward_threshold=defaults.THRESHOLD)
redis_ip = s3_client.get_ip()
logger.info("Received IP from SageMaker successfully: %s" % redis_ip)
# Download hyperparameters from SageMaker
hyperparameters_file_success = False
hyperparams_s3_key = os.path.normpath(args.s3_prefix +
"/ip/hyperparameters.json")
hyperparameters_file_success = s3_client.download_file(
s3_key=hyperparams_s3_key, local_path="hyperparameters.json")
sm_hyperparams_dict = {}
if hyperparameters_file_success:
logger.info("Received Sagemaker hyperparameters successfully!")
with open("hyperparameters.json") as fp:
sm_hyperparams_dict = json.load(fp)
else:
logger.info("SageMaker hyperparameters not found.")
preset_file_success, _ = download_custom_files_if_present(
s3_client, args.s3_prefix)
if preset_file_success:
preset_location = os.path.join(CUSTOM_FILES_PATH, "preset.py")
preset_location += ":graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
logger.info("Using custom preset file!")
else:
from markov.sagemaker_graph_manager import get_graph_manager
graph_manager, _ = get_graph_manager(**sm_hyperparams_dict)
logger.info("Connecting to redis at %s:%d" % (redis_ip, args.redis_port))
memory_backend_params = RedisPubSubMemoryBackendParameters(
redis_address=redis_ip,
redis_port=6379,
run_type='worker',
channel=args.s3_prefix)
logger.info("Connecting to s3 boto data store at %s" %
args.aws_endpoint_url)
ds_params_instance = S3BotoDataStoreParameters(
bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
aws_region=args.aws_region,
s3_folder=args.s3_prefix,
aws_endpoint_url=args.aws_endpoint_url)
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
rollout_worker(graph_manager=graph_manager,
checkpoint_dir=args.checkpoint_dir,
data_store=data_store,
num_workers=args.num_workers,
memory_backend_params=memory_backend_params)
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument('-pk',
'--preset_s3_key',
help="(string) Name of a preset to download from S3",
type=str,
required=False)
parser.add_argument(
'-ek',
'--environment_s3_key',
help="(string) Name of an environment file to download from S3",
type=str,
required=False)
parser.add_argument('--model_metadata_s3_key',
help="(string) Model Metadata File S3 Key",
type=str,
required=False)
parser.add_argument(
'-c',
'--checkpoint-dir',
help=
'(string) Path to a folder containing a checkpoint to write the model to.',
type=str,
default='./checkpoint')
parser.add_argument(
'--pretrained-checkpoint-dir',
help='(string) Path to a folder for downloading a pre-trained model',
type=str,
default=PRETRAINED_MODEL_DIR)
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get(
"SAGEMAKER_SHARED_S3_BUCKET_PATH", "gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default='sagemaker')
parser.add_argument('--framework',
help='(string) tensorflow or mxnet',
type=str,
default='tensorflow')
parser.add_argument('--pretrained_s3_bucket',
help='(string) S3 bucket for pre-trained model',
type=str)
parser.add_argument('--pretrained_s3_prefix',
help='(string) S3 prefix for pre-trained model',
type=str,
default='sagemaker')
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("AWS_REGION", "us-east-1"))
args, unknown = parser.parse_known_args()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
s3_client.upload_file(
os.path.normpath("%s/model/model_metadata.json" % args.s3_prefix),
model_metadata_local_path)
shutil.copy2(model_metadata_local_path, SM_MODEL_OUTPUT_DIR)
success_custom_environment = False
if args.environment_s3_key:
environment_local_path = "./markov/environments/deepracer_racetrack_env.py"
success_custom_environment = s3_client.download_file(
s3_key=args.environment_s3_key, local_path=environment_local_path)
if not success_custom_environment:
print(
"Could not download the environment file. Using the default DeepRacer environment."
)
else:
success_custom_environment = s3_client.upload_file(
s3_key=os.path.normpath(
"%s/environments/deepracer_racetrack_env.py" %
args.s3_prefix),
local_path=environment_local_path)
if success_custom_environment:
print("Using environment: %s" % args.environment_s3_key)
# Import to register the environment with Gym
import markov.environments
success_custom_preset = False
if args.preset_s3_key:
preset_local_path = "./markov/presets/preset.py"
success_custom_preset = s3_client.download_file(
s3_key=args.preset_s3_key, local_path=preset_local_path)
if not success_custom_preset:
print(
"Could not download the preset file. Using the default DeepRacer preset."
)
else:
preset_location = "markov.presets.preset:graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
success_custom_preset = s3_client.upload_file(
s3_key=os.path.normpath("%s/presets/preset.py" %
args.s3_prefix),
local_path=preset_local_path)
if success_custom_preset:
print("Using preset: %s" % args.preset_s3_key)
if not success_custom_preset:
from markov.sagemaker_graph_manager import get_graph_manager
params_blob = os.environ.get('SM_TRAINING_ENV', '')
if params_blob:
params = json.loads(params_blob)
sm_hyperparams_dict = params["hyperparameters"]
else:
sm_hyperparams_dict = {}
graph_manager, robomaker_hyperparams_json = get_graph_manager(
**sm_hyperparams_dict)
s3_client.upload_hyperparameters(robomaker_hyperparams_json)
print("Uploaded hyperparameters.json to S3")
host_ip_address = get_ip_from_host()
s3_client.write_ip_config(host_ip_address)
print("Uploaded IP address information to S3: %s" % host_ip_address)
use_pretrained_model = False
if args.pretrained_s3_bucket and args.pretrained_s3_prefix:
s3_client_pretrained = SageS3Client(
bucket=args.pretrained_s3_bucket,
s3_prefix=args.pretrained_s3_prefix,
aws_region=args.aws_region)
s3_client_pretrained.download_model(args.pretrained_checkpoint_dir)
use_pretrained_model = True
memory_backend_params = RedisPubSubMemoryBackendParameters(
redis_address="localhost",
redis_port=6379,
run_type='trainer',
channel=args.s3_prefix)
graph_manager.agent_params.memory.register_var('memory_backend_params',
memory_backend_params)
ds_params_instance = S3BotoDataStoreParameters(
bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
aws_region=args.aws_region,
s3_folder=args.s3_prefix)
graph_manager.data_store_params = ds_params_instance
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
training_worker(graph_manager=graph_manager,
checkpoint_dir=args.checkpoint_dir,
use_pretrained_model=use_pretrained_model,
framework=args.framework)
def main():
screen.set_use_colors(False)
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--checkpoint_dir',
help=
'(string) Path to a folder containing a checkpoint to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--s3_bucket',
help='(string) S3 bucket',
type=str,
default=os.environ.get("SAGEMAKER_SHARED_S3_BUCKET",
"gsaur-test"))
parser.add_argument('--s3_prefix',
help='(string) S3 prefix',
type=str,
default=os.environ.get("SAGEMAKER_SHARED_S3_PREFIX",
"sagemaker"))
parser.add_argument(
'--num-workers',
help="(int) The number of workers started in this pool",
type=int,
default=1)
parser.add_argument('-r',
'--redis_ip',
help="(string) IP or host for the redis server",
default='localhost',
type=str)
parser.add_argument('-rp',
'--redis_port',
help="(int) Port of the redis server",
default=6379,
type=int)
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=os.environ.get("APP_REGION", "us-east-1"))
parser.add_argument('--reward_file_s3_key',
help='(string) Reward File S3 Key',
type=str,
default=os.environ.get("REWARD_FILE_S3_KEY", None))
parser.add_argument('--model_metadata_s3_key',
help='(string) Model Metadata File S3 Key',
type=str,
default=os.environ.get("MODEL_METADATA_FILE_S3_KEY",
None))
args = parser.parse_args()
s3_client = SageS3Client(bucket=args.s3_bucket,
s3_prefix=args.s3_prefix,
aws_region=args.aws_region)
print("S3 bucket: %s" % args.s3_bucket)
print("S3 prefix: %s" % args.s3_prefix)
# Load the model metadata
model_metadata_local_path = os.path.join(CUSTOM_FILES_PATH,
'model_metadata.json')
load_model_metadata(s3_client, args.model_metadata_s3_key,
model_metadata_local_path)
redis_ip = s3_client.get_ip()
print("Received IP from SageMaker successfully: %s" % redis_ip)
# Download hyperparameters from SageMaker
hyperparameters_file_success = False
hyperparams_s3_key = os.path.normpath(args.s3_prefix +
"/ip/hyperparameters.json")
hyperparameters_file_success = s3_client.download_file(
s3_key=hyperparams_s3_key, local_path="hyperparameters.json")
sm_hyperparams_dict = {}
if hyperparameters_file_success:
print("Received Sagemaker hyperparameters successfully!")
with open("hyperparameters.json") as fp:
sm_hyperparams_dict = json.load(fp)
else:
print("SageMaker hyperparameters not found.")
preset_file_success = False
environment_file_success = False
preset_file_success, environment_file_success = download_custom_files_if_present(
s3_client, args.s3_prefix)
if not environment_file_success:
# Download reward function if environment file is not downloaded
if not args.reward_file_s3_key:
raise ValueError("Customer reward S3 key not supplied!")
download_customer_reward_function(s3_client, args.reward_file_s3_key)
import markov.environments
print("Using default environment!")
else:
register_custom_environments()
print("Using custom environment!")
if preset_file_success:
preset_location = os.path.join(CUSTOM_FILES_PATH, "preset.py")
preset_location += ":graph_manager"
graph_manager = short_dynamic_import(preset_location,
ignore_module_case=True)
print("Using custom preset file!")
else:
from markov.sagemaker_graph_manager import get_graph_manager
graph_manager, _ = get_graph_manager(**sm_hyperparams_dict)
memory_backend_params = RedisPubSubMemoryBackendParameters(
redis_address=redis_ip,
redis_port=6379,
run_type='worker',
channel=args.s3_prefix)
graph_manager.agent_params.memory.register_var('memory_backend_params',
memory_backend_params)
ds_params_instance = S3BotoDataStoreParameters(
bucket_name=args.s3_bucket,
checkpoint_dir=args.checkpoint_dir,
aws_region=args.aws_region,
s3_folder=args.s3_prefix)
data_store = S3BotoDataStore(ds_params_instance)
data_store.graph_manager = graph_manager
graph_manager.data_store = data_store
rollout_worker(
graph_manager=graph_manager,
checkpoint_dir=args.checkpoint_dir,
data_store=data_store,
num_workers=args.num_workers,
)