def _update_body_shell(self):
'''update body shell type in yaml dict and then overwrite local yaml file
with updated body shell type. This is for backward compatibility when
body shell is not pasted in for F1 with specific racer alias or
display name'''
# List of body shell types
body_shell_types = force_list(self._yaml_values.get(YamlKey.BODY_SHELL_TYPE_YAML_KEY.value, [None]))
racer_names = force_list(self._yaml_values.get(YamlKey.RACER_NAME_YAML_KEY.value, [None]))
display_names = force_list(self._yaml_values.get(YamlKey.DISPLAY_NAME_YAML_KEY.value, [None]))
# If body_shell_types contains only None, figure out shell based on names
# otherwise use body_shell_type defined in body_shell_types
if None in body_shell_types:
# use default shells only if both RACER_NAME and DISPLAY_NAME are empty
if None in racer_names and None in display_names:
body_shell_types = [BodyShellType.DEFAULT.value] *\
len(force_list(self._yaml_values[self._model_s3_bucket_yaml_key]))
else:
# If RACER_NAME is empty, use DISPLAY_NAME to get racer_alias,
# and check racer_alias in F1_SHELL_USERS_LIST whether to use F1 shell or not,
# otherwise use RACER_NAME as racer_alias to figure out
# whether to use the f1 body shell.
if None in racer_names:
body_shell_types = [BodyShellType.F1_2021.value if racer_alias
in F1_SHELL_USERS_LIST else BodyShellType.DEFAULT.value
for racer_alias in display_names]
else:
body_shell_types = [BodyShellType.F1_2021.value if racer_alias
in F1_SHELL_USERS_LIST else BodyShellType.DEFAULT.value
for racer_alias in racer_names]
self._yaml_values[YamlKey.BODY_SHELL_TYPE_YAML_KEY.value] = body_shell_types
# override local yaml file with updated BODY_SHELL_TYPE
self._overwrite_local_yaml_file()
def __init__(self, horizontal_fov, padding_percent, image_width,
image_height, racecars_info):
""" Camera information is required to map the (x, y) value of the agent on the camera image.
This is because each track will have its own FOV and padding percent because to avoid
Z-fighting. Once the camera position, padding percentage is available. We can map the
(x,y) of the agent with respect to the track to a new plane of the camera image with
padding.
Arguments:
horizontal_fov (float): Horizontal field of view of the camera for the given track
padding_percent (float): The padding percentage to cover the whole track and look pretty
image_width (int): Image width from the camera
image_height (int): Image width from the camera
racecars_info (list): This is the list of dicts of racecars on the track
"""
self.horizontal_fov = horizontal_fov
self.padding_percent = padding_percent
self.image_width = image_width
self.image_height = image_height
self.racecars_info = force_list(racecars_info)
self.model_imgs = self._get_all_models_info()
# Track information is required get the track bounds (xmin, xmax, ymin, ymax)
# Also the agent location for that instant
self.track_data = TrackData.get_instance()
# This maps the (x, y) of an agent in track frame to the top camera image frame
self.initialize_parameters()
def test_force_list(s3_bucket):
"""This function checks the functionality of force_list function
in markov/utils.py
Args:
s3_bucket (String): S3_BUCKET
"""
assert utils.force_list(s3_bucket) == [s3_bucket]
def get_racecars_info(racecar_names):
""" This function returns the agents information like name, car color, display name
Arguments:
racecar_names (str): comma seperated racecar names
Returns:
(list): Racecar information such as name, car color, display name
"""
racecars = racecar_names.split(',')
racecars_info = list()
racecars_color = force_list(rospy.get_param("CAR_COLOR", DEFAULT_COLOR))
racecars_display_name = force_list(rospy.get_param("DISPLAY_NAME", ""))
for i, racecar_name in enumerate(racecars):
racecar_dict = dict()
racecar_dict['name'] = racecar_name
racecar_dict['racecar_color'] = RaceCarColorToRGB[
racecars_color[i]].value
racecar_dict['display_name'] = racecars_display_name[i]
racecars_info.append(racecar_dict)
return racecars_info
def _postprocess_yaml_values(self):
'''postprocess yaml values
First, it will force all FORCE_LIST_PARAMS to a list.
Second, MODEL_METADATA_FILE_S3_YAML_KEY does not exist in eval,
so we have to manually write it.
'''
# Forcing all mandatory yaml param to list
for params in self._mandatory_yaml_key:
if params in self._yaml_values:
self._yaml_values[params] = force_list(
self._yaml_values[params])
#TODO: delete this logic if cloud service team can always pass model metadata
# populate the model_metadata_s3_key values to handle both
# training and evaluation for all race_formats
if YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value not in self._yaml_values:
# MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during
# evaluation Assume model_metadata.json is present in the s3_prefix/model/ folder
self._yaml_values[
YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value] = list()
for s3_prefix in self._yaml_values[self._model_s3_prefix_yaml_key]:
self._yaml_values[
YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value].append(
os.path.join(s3_prefix, MODEL_METADATA_S3_POSTFIX))
# set model s3 buckets
self._model_s3_buckets = self._yaml_values[
self._model_s3_bucket_yaml_key]
# set model metadata
self._model_metadata_s3_keys = \
self._yaml_values[YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value]
# set body shell types
self._body_shell_types = self._yaml_values[
YamlKey.BODY_SHELL_TYPE_YAML_KEY.value]
# set multicar value if there is more than one value in self._model_s3_bucket_yaml_key.
self._is_multicar = len(
self._yaml_values[self._model_s3_bucket_yaml_key]) > 1
# set f1 as true if RACE_TYPE is F1
self._is_f1 = self._yaml_values.get(
YamlKey.RACE_TYPE_YAML_KEY.value,
RaceType.TIME_TRIAL.value) == F1_RACE_TYPE
def __init__(self, horizontal_fov, padding_percent, image_width, image_height, racecars_info):
""" Camera information is required to map the (x, y) value of the agent on the camera image.
This is because each track will have its own FOV and padding percent because to avoid
Z-fighting. Once the camera position, padding percentage is available. We can map the
(x,y) of the agent with respect to the track to a new plane of the camera image with
padding.
Arguments:
horizontal_fov (float): Horizontal field of view of the camera for the given track
padding_percent (float): The padding percentage to cover the whole track and look pretty
image_width (int): Image width from the camera
image_height (int): Image width from the camera
racecars_info (list): This is the list of dicts of racecars on the track
"""
self.horizontal_fov = horizontal_fov
self.padding_percent = padding_percent
self.image_width = image_width
self.image_height = image_height
self.racecars_info = force_list(racecars_info)
rospy.wait_for_service('/gazebo/get_model_state')
# We have no guarantees as to when gazebo will load the model, therefore we need
# to wait until the model is loaded prior to resetting it for the first time
self.get_model_client = ServiceProxyWrapper('/gazebo/get_model_state', GetModelState)
self.model_names, self.model_imgs = self._get_all_models_info()
wait_for_model = True
while wait_for_model:
# If the model is not loaded the get model service will log an error
# Therefore, we add an timer to prevent the log from getting spammed with
# errors
time.sleep(WAIT_TO_PREVENT_SPAM)
model_status = list()
for model_name in self.model_names:
model = self.get_model_client(model_name, '')
model_status.append(model.success)
wait_for_model = not all(model_status)
# Track information is required get the track bounds (xmin, xmax, ymin, ymax)
# Also the agent location for that instant
self.track_data = TrackData.get_instance()
# This maps the (x, y) of an agent in track frame to the top camera image frame
self.initialize_parameters()
def get_racecars_info(racecar_names):
""" This function returns the agents information like name, car color, display name
Arguments:
racecar_names (list): comma seperated racecar names
Returns:
(list): Racecar information such as name, car color, display name
"""
racecars = racecar_names
racecars_info = list()
racecars_color = force_list(
rospy.get_param("CAR_COLOR", [DEFAULT_COLOR] * len(racecar_names)))
racecars_display_name = get_video_display_name()
for i, racecar_name in enumerate(racecars):
racecar_dict = dict()
racecar_dict['name'] = racecar_name
racecar_dict['racecar_color'] = racecars_color[i]
racecar_dict['display_name'] = racecars_display_name[i]
racecars_info.append(racecar_dict)
return racecars_info
def __init__(self, horizontal_fov, padding_percent, image_width,
image_height, racecars_info):
""" Camera information is required to map the (x, y) value of the agent on the camera image.
This is because each track will have its own FOV and padding percent because to avoid
Z-fighting. Once the camera position, padding percentage is available. We can map the
(x,y) of the agent with respect to the track to a new plane of the camera image with
padding.
Arguments:
horizontal_fov (float): Horizontal field of view of the camera for the given track
padding_percent (float): The padding percentage to cover the whole track and look pretty
image_width (int): Image width from the camera
image_height (int): Image width from the camera
racecars_info (list): This is the list of dicts of racecars on the track
"""
self.horizontal_fov = horizontal_fov
self.padding_percent = padding_percent
self.image_width = image_width
self.image_height = image_height
self.racecars_info = force_list(racecars_info)
#
# We have no guarantees as to when gazebo will load the model, therefore we need
# to wait until the model is loaded and markov packages has spawned all the models
#
rospy.wait_for_service('/gazebo/get_model_state')
rospy.wait_for_service('/robomaker_markov_package_ready')
self.get_model_client = ServiceProxyWrapper('/gazebo/get_model_state',
GetModelState)
self.model_names, self.model_imgs = self._get_all_models_info()
# Track information is required get the track bounds (xmin, xmax, ymin, ymax)
# Also the agent location for that instant
self.track_data = TrackData.get_instance()
# This maps the (x, y) of an agent in track frame to the top camera image frame
self.initialize_parameters()
def main():
""" Main function for tournament worker """
parser = argparse.ArgumentParser()
parser.add_argument('-p',
'--preset',
help="(string) Name of a preset to run \
(class name from the 'presets' directory.)",
type=str,
required=False)
parser.add_argument('--s3_bucket',
help='list(string) S3 bucket',
type=str,
nargs='+',
default=rospy.get_param("MODEL_S3_BUCKET",
["gsaur-test"]))
parser.add_argument('--s3_prefix',
help='list(string) S3 prefix',
type=str,
nargs='+',
default=rospy.get_param("MODEL_S3_PREFIX",
["sagemaker"]))
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=rospy.get_param("AWS_REGION", "us-east-1"))
parser.add_argument('--number_of_trials',
help='(integer) Number of trials',
type=int,
default=int(rospy.get_param("NUMBER_OF_TRIALS", 10)))
parser.add_argument(
'-c',
'--local_model_directory',
help='(string) Path to a folder containing a checkpoint \
to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--number_of_resets',
help='(integer) Number of resets',
type=int,
default=int(rospy.get_param("NUMBER_OF_RESETS", 0)))
parser.add_argument('--penalty_seconds',
help='(float) penalty second',
type=float,
default=float(rospy.get_param("PENALTY_SECONDS", 2.0)))
parser.add_argument('--job_type',
help='(string) job type',
type=str,
default=rospy.get_param("JOB_TYPE", "EVALUATION"))
parser.add_argument('--is_continuous',
help='(boolean) is continous after lap completion',
type=bool,
default=utils.str2bool(
rospy.get_param("IS_CONTINUOUS", False)))
parser.add_argument('--race_type',
help='(string) Race type',
type=str,
default=rospy.get_param("RACE_TYPE", "TIME_TRIAL"))
parser.add_argument('--off_track_penalty',
help='(float) off track penalty second',
type=float,
default=float(rospy.get_param("OFF_TRACK_PENALTY",
2.0)))
parser.add_argument('--collision_penalty',
help='(float) collision penalty second',
type=float,
default=float(rospy.get_param("COLLISION_PENALTY",
5.0)))
args = parser.parse_args()
arg_s3_bucket = args.s3_bucket
arg_s3_prefix = args.s3_prefix
logger.info("S3 bucket: %s \n S3 prefix: %s", arg_s3_bucket, arg_s3_prefix)
# tournament_worker: names to be displayed in MP4.
# This is racer alias in tournament worker case.
display_names = rospy.get_param('DISPLAY_NAME', "")
metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET')
metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY')
arg_s3_bucket, arg_s3_prefix = utils.force_list(
arg_s3_bucket), utils.force_list(arg_s3_prefix)
metrics_s3_buckets = utils.force_list(metrics_s3_buckets)
metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys)
validate_list = [
arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets,
metrics_s3_object_keys
]
simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None)
mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None)
if simtrace_s3_bucket:
simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX')
simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket)
simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix)
validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix])
if mp4_s3_bucket:
mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX')
mp4_s3_bucket = utils.force_list(mp4_s3_bucket)
mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix)
validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix])
if not all([lambda x: len(x) == len(validate_list[0]), validate_list]):
utils.log_and_exit(
"Eval worker error: Incorrect arguments passed: {}".format(
validate_list), utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500)
if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT:
raise GenericRolloutException(
"number of resets is less than {}".format(MIN_RESET_COUNT))
# Instantiate Cameras
if len(arg_s3_bucket) == 1:
configure_camera(namespaces=['racecar'])
else:
configure_camera(namespaces=[
'racecar_{}'.format(str(agent_index))
for agent_index in range(len(arg_s3_bucket))
])
agent_list = list()
s3_bucket_dict = dict()
s3_prefix_dict = dict()
s3_writers = list()
# tournament_worker: list of required S3 locations
simtrace_s3_bucket_dict = dict()
simtrace_s3_prefix_dict = dict()
metrics_s3_bucket_dict = dict()
metrics_s3_obect_key_dict = dict()
mp4_s3_bucket_dict = dict()
mp4_s3_object_prefix_dict = dict()
for agent_index, s3_bucket_val in enumerate(arg_s3_bucket):
agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format(
str(agent_index))
racecar_name = 'racecar' if len(
arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index))
s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index]
s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index]
# tournament_worker: remap key with agent_name instead of agent_index for list of S3 locations.
simtrace_s3_bucket_dict[agent_name] = simtrace_s3_bucket[agent_index]
simtrace_s3_prefix_dict[agent_name] = simtrace_s3_object_prefix[
agent_index]
metrics_s3_bucket_dict[agent_name] = metrics_s3_buckets[agent_index]
metrics_s3_obect_key_dict[agent_name] = metrics_s3_object_keys[
agent_index]
mp4_s3_bucket_dict[agent_name] = mp4_s3_bucket[agent_index]
mp4_s3_object_prefix_dict[agent_name] = mp4_s3_object_prefix[
agent_index]
s3_client = SageS3Client(bucket=arg_s3_bucket[agent_index],
s3_prefix=arg_s3_prefix[agent_index],
aws_region=args.aws_region)
# Load the model metadata
if not os.path.exists(os.path.join(CUSTOM_FILES_PATH, agent_name)):
os.makedirs(os.path.join(CUSTOM_FILES_PATH, agent_name))
model_metadata_local_path = os.path.join(
os.path.join(CUSTOM_FILES_PATH, agent_name), 'model_metadata.json')
utils.load_model_metadata(
s3_client,
os.path.normpath("%s/model/model_metadata.json" %
arg_s3_prefix[agent_index]),
model_metadata_local_path)
# Handle backward compatibility
_, _, version = parse_model_metadata(model_metadata_local_path)
if float(version) < float(utils.SIMAPP_VERSION) and \
not utils.has_current_ckpnt_name(arg_s3_bucket[agent_index], arg_s3_prefix[agent_index], args.aws_region):
utils.make_compatible(arg_s3_bucket[agent_index],
arg_s3_prefix[agent_index], args.aws_region,
SyncFiles.TRAINER_READY.value)
# Select the optimal model
utils.do_model_selection(s3_bucket=arg_s3_bucket[agent_index],
s3_prefix=arg_s3_prefix[agent_index],
region=args.aws_region)
# Download hyperparameters from SageMaker
if not os.path.exists(agent_name):
os.makedirs(agent_name)
hyperparameters_file_success = False
hyperparams_s3_key = os.path.normpath(arg_s3_prefix[agent_index] +
"/ip/hyperparameters.json")
hyperparameters_file_success = s3_client.download_file(
s3_key=hyperparams_s3_key,
local_path=os.path.join(agent_name, "hyperparameters.json"))
sm_hyperparams_dict = {}
if hyperparameters_file_success:
logger.info("Received Sagemaker hyperparameters successfully!")
with open(os.path.join(agent_name,
"hyperparameters.json")) as file:
sm_hyperparams_dict = json.load(file)
else:
logger.info("SageMaker hyperparameters not found.")
agent_config = {
'model_metadata': model_metadata_local_path,
ConfigParams.CAR_CTRL_CONFIG.value: {
ConfigParams.LINK_NAME_LIST.value: [
link_name.replace('racecar', racecar_name)
for link_name in LINK_NAMES
],
ConfigParams.VELOCITY_LIST.value: [
velocity_topic.replace('racecar', racecar_name)
for velocity_topic in VELOCITY_TOPICS
],
ConfigParams.STEERING_LIST.value: [
steering_topic.replace('racecar', racecar_name)
for steering_topic in STEERING_TOPICS
],
ConfigParams.CHANGE_START.value:
utils.str2bool(rospy.get_param('CHANGE_START_POSITION',
False)),
ConfigParams.ALT_DIR.value:
utils.str2bool(
rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)),
ConfigParams.ACTION_SPACE_PATH.value:
'custom_files/' + agent_name + '/model_metadata.json',
ConfigParams.REWARD.value:
reward_function,
ConfigParams.AGENT_NAME.value:
racecar_name,
ConfigParams.VERSION.value:
version,
ConfigParams.NUMBER_OF_RESETS.value:
args.number_of_resets,
ConfigParams.PENALTY_SECONDS.value:
args.penalty_seconds,
ConfigParams.NUMBER_OF_TRIALS.value:
args.number_of_trials,
ConfigParams.IS_CONTINUOUS.value:
args.is_continuous,
ConfigParams.RACE_TYPE.value:
args.race_type,
ConfigParams.COLLISION_PENALTY.value:
args.collision_penalty,
ConfigParams.OFF_TRACK_PENALTY.value:
args.off_track_penalty
}
}
metrics_s3_config = {
MetricsS3Keys.METRICS_BUCKET.value:
metrics_s3_buckets[agent_index],
MetricsS3Keys.METRICS_KEY.value:
metrics_s3_object_keys[agent_index],
# Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed
# or default argument set
MetricsS3Keys.REGION.value:
args.aws_region,
# Replaced rospy.get_param('MODEL_S3_BUCKET') to be equal to the argument being passed
# or default argument set
MetricsS3Keys.STEP_BUCKET.value:
arg_s3_bucket[agent_index],
# Replaced rospy.get_param('MODEL_S3_PREFIX') to be equal to the argument being passed
# or default argument set
MetricsS3Keys.STEP_KEY.value:
os.path.join(arg_s3_prefix[agent_index],
EVALUATION_SIMTRACE_DATA_S3_OBJECT_KEY)
}
aws_region = rospy.get_param('AWS_REGION', args.aws_region)
s3_writer_job_info = []
if simtrace_s3_bucket:
s3_writer_job_info.append(
IterationData(
'simtrace', simtrace_s3_bucket[agent_index],
simtrace_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
SIM_TRACE_EVALUATION_LOCAL_FILE.value)))
if mp4_s3_bucket:
s3_writer_job_info.extend([
IterationData(
'pip', mp4_s3_bucket[agent_index],
mp4_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
CAMERA_PIP_MP4_VALIDATION_LOCAL_PATH.value)),
IterationData(
'45degree', mp4_s3_bucket[agent_index],
mp4_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
CAMERA_45DEGREE_MP4_VALIDATION_LOCAL_PATH.value)),
IterationData(
'topview', mp4_s3_bucket[agent_index],
mp4_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
CAMERA_TOPVIEW_MP4_VALIDATION_LOCAL_PATH.value))
])
s3_writers.append(S3Writer(job_info=s3_writer_job_info))
run_phase_subject = RunPhaseSubject()
agent_list.append(
create_rollout_agent(agent_config,
EvalMetrics(agent_name, metrics_s3_config),
run_phase_subject))
agent_list.append(create_obstacles_agent())
agent_list.append(create_bot_cars_agent())
# ROS service to indicate all the robomaker markov packages are ready for consumption
signal_robomaker_markov_package_ready()
PhaseObserver('/agent/training_phase', run_phase_subject)
graph_manager, _ = get_graph_manager(hp_dict=sm_hyperparams_dict,
agent_list=agent_list,
run_phase_subject=run_phase_subject)
ds_params_instance = S3BotoDataStoreParameters(
aws_region=args.aws_region,
bucket_names=s3_bucket_dict,
base_checkpoint_dir=args.local_model_directory,
s3_folders=s3_prefix_dict)
graph_manager.data_store = S3BotoDataStore(params=ds_params_instance,
graph_manager=graph_manager,
ignore_lock=True)
graph_manager.env_params.seed = 0
task_parameters = TaskParameters()
task_parameters.checkpoint_restore_path = args.local_model_directory
tournament_worker(graph_manager=graph_manager,
number_of_trials=args.number_of_trials,
task_parameters=task_parameters,
s3_writers=s3_writers,
is_continuous=args.is_continuous)
# tournament_worker: write race report to local file.
write_race_report(graph_manager,
model_s3_bucket_map=s3_bucket_dict,
model_s3_prefix_map=s3_prefix_dict,
metrics_s3_bucket_map=metrics_s3_bucket_dict,
metrics_s3_key_map=metrics_s3_obect_key_dict,
simtrace_s3_bucket_map=simtrace_s3_bucket_dict,
simtrace_s3_prefix_map=simtrace_s3_prefix_dict,
mp4_s3_bucket_map=mp4_s3_bucket_dict,
mp4_s3_prefix_map=mp4_s3_object_prefix_dict,
display_names=display_names)
# tournament_worker: terminate tournament_race_node.
terminate_tournament_race()
def main():
""" Main function for tournament"""
try:
# parse argument
s3_region = sys.argv[1]
s3_bucket = sys.argv[2]
s3_prefix = sys.argv[3]
s3_yaml_name = sys.argv[4]
# create boto3 session/client and download yaml/json file
session = boto3.session.Session()
s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None)
s3_client = session.client('s3',
region_name=s3_region,
endpoint_url=s3_endpoint_url,
config=get_boto_config())
yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name))
local_yaml_path = os.path.abspath(
os.path.join(os.getcwd(), s3_yaml_name))
s3_client.download_file(Bucket=s3_bucket,
Key=yaml_key,
Filename=local_yaml_path)
# Intermediate tournament files
queue_pickle_name = 'tournament_candidate_queue.pkl'
queue_pickle_s3_key = os.path.normpath(
os.path.join(s3_prefix, queue_pickle_name))
local_queue_pickle_path = os.path.abspath(
os.path.join(os.getcwd(), queue_pickle_name))
report_pickle_name = 'tournament_report.pkl'
report_pickle_s3_key = os.path.normpath(
os.path.join(s3_prefix, report_pickle_name))
local_report_pickle_path = os.path.abspath(
os.path.join(os.getcwd(), report_pickle_name))
final_report_name = 'tournament_report.json'
final_report_s3_key = os.path.normpath(
os.path.join(s3_prefix, final_report_name))
try:
s3_client.download_file(Bucket=s3_bucket,
Key=queue_pickle_s3_key,
Filename=local_queue_pickle_path)
s3_client.download_file(Bucket=s3_bucket,
Key=report_pickle_s3_key,
Filename=local_report_pickle_path)
except:
pass
# Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing
yaml_dict = get_yaml_dict(local_yaml_path)
# Forcing the yaml parameter to list
force_list_params = [
MODEL_S3_BUCKET_YAML_KEY, MODEL_S3_PREFIX_YAML_KEY,
MODEL_METADATA_FILE_S3_YAML_KEY, METRICS_S3_BUCKET_YAML_KEY,
METRICS_S3_PREFIX_YAML_KEY, SIMTRACE_S3_BUCKET_YAML_KEY,
SIMTRACE_S3_PREFIX_YAML_KEY, MP4_S3_BUCKET_YAML_KEY,
MP4_S3_PREFIX_YAML_KEY, DISPLAY_NAME_YAML_KEY
]
for params in force_list_params:
yaml_dict[params] = force_list(yaml_dict.get(params, None))
# Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats
if None in yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY]:
# MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation
# Assume model_metadata.json is present in the s3_prefix/model/ folder
yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY] = list()
for s3_prefix in yaml_dict[MODEL_S3_PREFIX_YAML_KEY]:
yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY].append(
os.path.join(s3_prefix, 'model/model_metadata.json'))
# Validate the yaml values
validate_yaml_values(yaml_dict)
if os.path.exists(local_queue_pickle_path):
with open(local_queue_pickle_path, 'rb') as f:
tournament_candidate_queue = pickle.load(f)
with open(local_report_pickle_path, 'rb') as f:
tournament_report = pickle.load(f)
logger.info('tournament_candidate_queue loaded from existing file')
else:
logger.info('tournament_candidate_queue initialized')
tournament_candidate_queue = deque()
for agent_idx, _ in enumerate(yaml_dict[MODEL_S3_BUCKET_YAML_KEY]):
tournament_candidate_queue.append(
(yaml_dict[MODEL_S3_BUCKET_YAML_KEY][agent_idx],
yaml_dict[MODEL_S3_PREFIX_YAML_KEY][agent_idx],
yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY][agent_idx],
yaml_dict[METRICS_S3_BUCKET_YAML_KEY][agent_idx],
yaml_dict[METRICS_S3_PREFIX_YAML_KEY][agent_idx],
yaml_dict[SIMTRACE_S3_BUCKET_YAML_KEY][agent_idx],
yaml_dict[SIMTRACE_S3_PREFIX_YAML_KEY][agent_idx],
yaml_dict[MP4_S3_BUCKET_YAML_KEY][agent_idx],
yaml_dict[MP4_S3_PREFIX_YAML_KEY][agent_idx],
yaml_dict[DISPLAY_NAME_YAML_KEY][agent_idx]))
tournament_report = []
race_idx = len(tournament_report)
while len(tournament_candidate_queue) > 1:
car1 = tournament_candidate_queue.popleft()
car2 = tournament_candidate_queue.popleft()
(car1_model_s3_bucket, car1_s3_prefix, car1_model_metadata,
car1_metrics_bucket, car1_metrics_s3_key, car1_simtrace_bucket,
car1_simtrace_prefix, car1_mp4_bucket, car1_mp4_prefix,
car1_display_name) = car1
(car2_model_s3_bucket, car2_s3_prefix, car2_model_metadata,
car2_metrics_bucket, car2_metrics_s3_key, car2_simtrace_bucket,
car2_simtrace_prefix, car2_mp4_bucket, car2_mp4_prefix,
car2_display_name) = car2
race_yaml_dict = generate_race_yaml(yaml_dict=yaml_dict,
car1=car1,
car2=car2,
race_idx=race_idx)
race_car_colors = ["Orange", "Purple"]
race_model_s3_buckets = [
car1_model_s3_bucket, car2_model_s3_bucket
]
race_model_metadatas = [car1_model_metadata, car2_model_metadata]
# List of directories created
dirs_to_delete = list()
yaml_dir = os.path.abspath(os.path.join(os.getcwd(),
str(race_idx)))
os.makedirs(yaml_dir)
dirs_to_delete.append(yaml_dir)
race_yaml_path = os.path.abspath(
os.path.join(yaml_dir, 'evaluation_params.yaml'))
with open(race_yaml_path, 'w') as race_yaml_file:
yaml.dump(race_yaml_dict, race_yaml_file)
# List of racecar names that should include second camera while launching
racecars_with_stereo_cameras = list()
# List of racecar names that should include lidar while launching
racecars_with_lidars = list()
# List of SimApp versions
simapp_versions = list()
for agent_index, model_s3_bucket in enumerate(
race_model_s3_buckets):
racecar_name = 'racecar_' + str(agent_index)
# Make a local folder with the racecar name to download the model_metadata.json
os.makedirs(os.path.join(os.getcwd(), racecar_name))
dirs_to_delete.append(os.path.join(os.getcwd(), racecar_name))
local_model_metadata_path = os.path.abspath(
os.path.join(os.path.join(os.getcwd(), racecar_name),
'model_metadata.json'))
json_key = race_model_metadatas[agent_index]
json_key = json_key.replace('s3://{}/'.format(model_s3_bucket),
'')
s3_client.download_file(Bucket=model_s3_bucket,
Key=json_key,
Filename=local_model_metadata_path)
sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata(
local_model_metadata_path)
simapp_versions.append(simapp_version)
if Input.STEREO.value in sensors:
racecars_with_stereo_cameras.append(racecar_name)
if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors:
racecars_with_lidars.append(racecar_name)
cmd = [
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"tournament_race_node.py"),
str(race_idx), race_yaml_path,
','.join(racecars_with_stereo_cameras),
','.join(racecars_with_lidars), ','.join(race_car_colors),
','.join(simapp_versions)
]
try:
return_code, _, stderr = run_cmd(cmd_args=cmd,
shell=False,
stdout=None,
stderr=None)
except KeyboardInterrupt:
logger.info(
"KeyboardInterrupt raised, SimApp must be faulted! exiting..."
)
return
# Retrieve winner and append tournament report
with open('race_report.pkl', 'rb') as f:
race_report = pickle.load(f)
race_report['race_idx'] = race_idx
winner = car1 if race_report[
'winner'] == car1_display_name else car2
logger.info("race {}'s winner: {}".format(race_idx,
race_report['winner']))
tournament_candidate_queue.append(winner)
tournament_report.append(race_report)
# Clean up directories created
for dir_to_delete in dirs_to_delete:
shutil.rmtree(dir_to_delete, ignore_errors=True)
race_idx += 1
# Persist latest queue and report to use after job restarts.
with open(local_queue_pickle_path, 'wb') as f:
pickle.dump(tournament_candidate_queue, f, protocol=2)
s3_client.upload_file(Filename=local_queue_pickle_path,
Bucket=s3_bucket,
Key=queue_pickle_s3_key)
with open(local_report_pickle_path, 'wb') as f:
pickle.dump(tournament_report, f, protocol=2)
s3_client.upload_file(Filename=local_report_pickle_path,
Bucket=s3_bucket,
Key=report_pickle_s3_key)
# If there is more than 1 candidates then restart the simulation job otherwise
# tournament is finished, persists final report and ends the job.
if len(tournament_candidate_queue) > 1:
restart_simulation_job(
os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'),
s3_region)
break
else:
# Persist final tournament report in json format
# and terminate the job by canceling it
s3_client.put_object(Bucket=s3_bucket,
Key=final_report_s3_key,
Body=json.dumps(tournament_report))
cancel_simulation_job(
os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'),
s3_region)
except Exception as e:
log_and_exit(
"Tournament node failed: s3_bucket: {}, yaml_key: {}, {}".format(
s3_bucket, yaml_key, e), SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def main():
""" Main function for evaluation worker """
parser = argparse.ArgumentParser()
parser.add_argument('-p',
'--preset',
help="(string) Name of a preset to run \
(class name from the 'presets' directory.)",
type=str,
required=False)
parser.add_argument('--s3_bucket',
help='list(string) S3 bucket',
type=str,
nargs='+',
default=rospy.get_param("MODEL_S3_BUCKET",
["gsaur-test"]))
parser.add_argument('--s3_prefix',
help='list(string) S3 prefix',
type=str,
nargs='+',
default=rospy.get_param("MODEL_S3_PREFIX",
["sagemaker"]))
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=rospy.get_param("AWS_REGION", "us-east-1"))
parser.add_argument('--number_of_trials',
help='(integer) Number of trials',
type=int,
default=int(rospy.get_param("NUMBER_OF_TRIALS", 10)))
parser.add_argument(
'-c',
'--local_model_directory',
help='(string) Path to a folder containing a checkpoint \
to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--number_of_resets',
help='(integer) Number of resets',
type=int,
default=int(rospy.get_param("NUMBER_OF_RESETS", 0)))
parser.add_argument('--penalty_seconds',
help='(float) penalty second',
type=float,
default=float(rospy.get_param("PENALTY_SECONDS", 2.0)))
parser.add_argument('--job_type',
help='(string) job type',
type=str,
default=rospy.get_param("JOB_TYPE", "EVALUATION"))
parser.add_argument('--is_continuous',
help='(boolean) is continous after lap completion',
type=bool,
default=utils.str2bool(
rospy.get_param("IS_CONTINUOUS", False)))
parser.add_argument('--race_type',
help='(string) Race type',
type=str,
default=rospy.get_param("RACE_TYPE", "TIME_TRIAL"))
parser.add_argument('--off_track_penalty',
help='(float) off track penalty second',
type=float,
default=float(rospy.get_param("OFF_TRACK_PENALTY",
2.0)))
parser.add_argument('--collision_penalty',
help='(float) collision penalty second',
type=float,
default=float(rospy.get_param("COLLISION_PENALTY",
5.0)))
args = parser.parse_args()
arg_s3_bucket = args.s3_bucket
arg_s3_prefix = args.s3_prefix
logger.info("S3 bucket: %s \n S3 prefix: %s", arg_s3_bucket, arg_s3_prefix)
metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET')
metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY')
arg_s3_bucket, arg_s3_prefix = utils.force_list(
arg_s3_bucket), utils.force_list(arg_s3_prefix)
metrics_s3_buckets = utils.force_list(metrics_s3_buckets)
metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys)
validate_list = [
arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets,
metrics_s3_object_keys
]
simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None)
mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None)
if simtrace_s3_bucket:
simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX')
simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket)
simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix)
validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix])
if mp4_s3_bucket:
mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX')
mp4_s3_bucket = utils.force_list(mp4_s3_bucket)
mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix)
validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix])
if not all([lambda x: len(x) == len(validate_list[0]), validate_list]):
log_and_exit(
"Eval worker error: Incorrect arguments passed: {}".format(
validate_list), SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT:
raise GenericRolloutException(
"number of resets is less than {}".format(MIN_RESET_COUNT))
# Instantiate Cameras
if len(arg_s3_bucket) == 1:
configure_camera(namespaces=['racecar'])
else:
configure_camera(namespaces=[
'racecar_{}'.format(str(agent_index))
for agent_index in range(len(arg_s3_bucket))
])
agent_list = list()
s3_bucket_dict = dict()
s3_prefix_dict = dict()
checkpoint_dict = dict()
simtrace_video_s3_writers = []
start_positions = get_start_positions(len(arg_s3_bucket))
done_condition = utils.str_to_done_condition(
rospy.get_param("DONE_CONDITION", any))
park_positions = utils.pos_2d_str_to_list(
rospy.get_param("PARK_POSITIONS", []))
# if not pass in park positions for all done condition case, use default
if not park_positions:
park_positions = [DEFAULT_PARK_POSITION for _ in arg_s3_bucket]
for agent_index, _ in enumerate(arg_s3_bucket):
agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format(
str(agent_index))
racecar_name = 'racecar' if len(
arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index))
s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index]
s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index]
# download model metadata
model_metadata = ModelMetadata(
bucket=arg_s3_bucket[agent_index],
s3_key=get_s3_key(arg_s3_prefix[agent_index],
MODEL_METADATA_S3_POSTFIX),
region_name=args.aws_region,
local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format(agent_name))
model_metadata_info = model_metadata.get_model_metadata_info()
version = model_metadata_info[ModelMetadataKeys.VERSION.value]
# checkpoint s3 instance
checkpoint = Checkpoint(bucket=arg_s3_bucket[agent_index],
s3_prefix=arg_s3_prefix[agent_index],
region_name=args.aws_region,
agent_name=agent_name,
checkpoint_dir=args.local_model_directory)
# make coach checkpoint compatible
if version < SIMAPP_VERSION_2 and not checkpoint.rl_coach_checkpoint.is_compatible(
):
checkpoint.rl_coach_checkpoint.make_compatible(
checkpoint.syncfile_ready)
# get best model checkpoint string
model_checkpoint_name = checkpoint.deepracer_checkpoint_json.get_deepracer_best_checkpoint(
)
# Select the best checkpoint model by uploading rl coach .coach_checkpoint file
checkpoint.rl_coach_checkpoint.update(
model_checkpoint_name=model_checkpoint_name,
s3_kms_extra_args=utils.get_s3_kms_extra_args())
checkpoint_dict[agent_name] = checkpoint
agent_config = {
'model_metadata': model_metadata,
ConfigParams.CAR_CTRL_CONFIG.value: {
ConfigParams.LINK_NAME_LIST.value: [
link_name.replace('racecar', racecar_name)
for link_name in LINK_NAMES
],
ConfigParams.VELOCITY_LIST.value: [
velocity_topic.replace('racecar', racecar_name)
for velocity_topic in VELOCITY_TOPICS
],
ConfigParams.STEERING_LIST.value: [
steering_topic.replace('racecar', racecar_name)
for steering_topic in STEERING_TOPICS
],
ConfigParams.CHANGE_START.value:
utils.str2bool(rospy.get_param('CHANGE_START_POSITION',
False)),
ConfigParams.ALT_DIR.value:
utils.str2bool(
rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)),
ConfigParams.MODEL_METADATA.value:
model_metadata,
ConfigParams.REWARD.value:
reward_function,
ConfigParams.AGENT_NAME.value:
racecar_name,
ConfigParams.VERSION.value:
version,
ConfigParams.NUMBER_OF_RESETS.value:
args.number_of_resets,
ConfigParams.PENALTY_SECONDS.value:
args.penalty_seconds,
ConfigParams.NUMBER_OF_TRIALS.value:
args.number_of_trials,
ConfigParams.IS_CONTINUOUS.value:
args.is_continuous,
ConfigParams.RACE_TYPE.value:
args.race_type,
ConfigParams.COLLISION_PENALTY.value:
args.collision_penalty,
ConfigParams.OFF_TRACK_PENALTY.value:
args.off_track_penalty,
ConfigParams.START_POSITION.value:
start_positions[agent_index],
ConfigParams.DONE_CONDITION.value:
done_condition
}
}
metrics_s3_config = {
MetricsS3Keys.METRICS_BUCKET.value:
metrics_s3_buckets[agent_index],
MetricsS3Keys.METRICS_KEY.value:
metrics_s3_object_keys[agent_index],
# Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed
# or default argument set
MetricsS3Keys.REGION.value:
args.aws_region
}
aws_region = rospy.get_param('AWS_REGION', args.aws_region)
if simtrace_s3_bucket:
simtrace_video_s3_writers.append(
SimtraceVideo(
upload_type=SimtraceVideoNames.SIMTRACE_EVAL.value,
bucket=simtrace_s3_bucket[agent_index],
s3_prefix=simtrace_s3_object_prefix[agent_index],
region_name=aws_region,
local_path=SIMTRACE_EVAL_LOCAL_PATH_FORMAT.format(
agent_name)))
if mp4_s3_bucket:
simtrace_video_s3_writers.extend([
SimtraceVideo(
upload_type=SimtraceVideoNames.PIP.value,
bucket=mp4_s3_bucket[agent_index],
s3_prefix=mp4_s3_object_prefix[agent_index],
region_name=aws_region,
local_path=CAMERA_PIP_MP4_LOCAL_PATH_FORMAT.format(
agent_name)),
SimtraceVideo(
upload_type=SimtraceVideoNames.DEGREE45.value,
bucket=mp4_s3_bucket[agent_index],
s3_prefix=mp4_s3_object_prefix[agent_index],
region_name=aws_region,
local_path=CAMERA_45DEGREE_LOCAL_PATH_FORMAT.format(
agent_name)),
SimtraceVideo(
upload_type=SimtraceVideoNames.TOPVIEW.value,
bucket=mp4_s3_bucket[agent_index],
s3_prefix=mp4_s3_object_prefix[agent_index],
region_name=aws_region,
local_path=CAMERA_TOPVIEW_LOCAL_PATH_FORMAT.format(
agent_name))
])
run_phase_subject = RunPhaseSubject()
agent_list.append(
create_rollout_agent(
agent_config,
EvalMetrics(agent_name, metrics_s3_config, args.is_continuous),
run_phase_subject))
agent_list.append(create_obstacles_agent())
agent_list.append(create_bot_cars_agent())
# ROS service to indicate all the robomaker markov packages are ready for consumption
signal_robomaker_markov_package_ready()
PhaseObserver('/agent/training_phase', run_phase_subject)
enable_domain_randomization = utils.str2bool(
rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False))
sm_hyperparams_dict = {}
# Make the clients that will allow us to pause and unpause the physics
rospy.wait_for_service('/gazebo/pause_physics_dr')
rospy.wait_for_service('/gazebo/unpause_physics_dr')
pause_physics = ServiceProxyWrapper('/gazebo/pause_physics_dr', Empty)
unpause_physics = ServiceProxyWrapper('/gazebo/unpause_physics_dr', Empty)
graph_manager, _ = get_graph_manager(
hp_dict=sm_hyperparams_dict,
agent_list=agent_list,
run_phase_subject=run_phase_subject,
enable_domain_randomization=enable_domain_randomization,
done_condition=done_condition,
pause_physics=pause_physics,
unpause_physics=unpause_physics)
ds_params_instance = S3BotoDataStoreParameters(
checkpoint_dict=checkpoint_dict)
graph_manager.data_store = S3BotoDataStore(params=ds_params_instance,
graph_manager=graph_manager,
ignore_lock=True)
graph_manager.env_params.seed = 0
task_parameters = TaskParameters()
task_parameters.checkpoint_restore_path = args.local_model_directory
evaluation_worker(graph_manager=graph_manager,
number_of_trials=args.number_of_trials,
task_parameters=task_parameters,
simtrace_video_s3_writers=simtrace_video_s3_writers,
is_continuous=args.is_continuous,
park_positions=park_positions,
race_type=args.race_type,
pause_physics=pause_physics,
unpause_physics=unpause_physics)
def main():
""" Main function for downloading yaml params """
try:
# parse argument
s3_region = sys.argv[1]
s3_bucket = sys.argv[2]
s3_prefix = sys.argv[3]
s3_yaml_name = sys.argv[4]
launch_name = sys.argv[5]
# create boto3 session/client and download yaml/json file
session = boto3.session.Session()
s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None)
if s3_endpoint_url is not None:
LOG.info('Endpoint URL {}'.format(s3_endpoint_url))
rospy.set_param('S3_ENDPOINT_URL', s3_endpoint_url)
else:
# create boto3 session/client and download yaml/json file
ec2_client = session.client('ec2', s3_region)
LOG.info('Checking internet connection...')
response = ec2_client.describe_vpcs()
if not response['Vpcs']:
log_and_exit("No VPC attached to instance",
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
LOG.info('Verified internet connection')
s3_client = session.client('s3',
region_name=s3_region,
endpoint_url=s3_endpoint_url,
config=get_boto_config())
yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name))
local_yaml_path = os.path.abspath(
os.path.join(os.getcwd(), s3_yaml_name))
s3_client.download_file(Bucket=s3_bucket,
Key=yaml_key,
Filename=local_yaml_path)
# Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing
default_yaml_values = {
RACE_TYPE_YAML_KEY: TIME_TRIAL_RACE_TYPE,
MODEL_S3_BUCKET_YAML_KEY: s3_bucket,
MODEL_S3_PREFIX_YAML_KEY: s3_prefix,
CAR_COLOR_YAML_KEY: DEFAULT_COLOR,
MODEL_METADATA_FILE_S3_YAML_KEY: None
}
yaml_dict = get_yaml_dict(local_yaml_path)
yaml_values = get_yaml_values(yaml_dict, default_yaml_values)
# Forcing the yaml parameter to list
force_list_params = [
MODEL_METADATA_FILE_S3_YAML_KEY, MODEL_S3_BUCKET_YAML_KEY,
MODEL_S3_PREFIX_YAML_KEY, CAR_COLOR_YAML_KEY
]
for params in force_list_params:
yaml_values[params] = force_list(yaml_values[params])
# Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats
if None in yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY]:
# MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation
# Assume model_metadata.json is present in the s3_prefix/model/ folder
yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY] = list()
for s3_prefix in yaml_values[MODEL_S3_PREFIX_YAML_KEY]:
yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY].append(
os.path.join(s3_prefix, 'model/model_metadata.json'))
# Set multicar value if its a head to model racetype
multicar = yaml_values[RACE_TYPE_YAML_KEY] == HEAD_TO_MODEL_RACE_TYPE
# Validate the yaml values
validate_yaml_values(yaml_values, multicar)
# List of racecar names that should include second camera while launching
racecars_with_stereo_cameras = list()
# List of racecar names that should include lidar while launching
racecars_with_lidars = list()
# List of SimApp versions
simapp_versions = list()
for agent_index, model_s3_bucket in enumerate(
yaml_values[MODEL_S3_BUCKET_YAML_KEY]):
racecar_name = 'racecar_' + str(agent_index) if len(
yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 else 'racecar'
# Make a local folder with the racecar name to download the model_metadata.json
if not os.path.exists(os.path.join(os.getcwd(), racecar_name)):
os.makedirs(os.path.join(os.getcwd(), racecar_name))
local_model_metadata_path = os.path.abspath(
os.path.join(os.path.join(os.getcwd(), racecar_name),
'model_metadata.json'))
json_key = yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY][
agent_index]
json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '')
s3_client.download_file(Bucket=model_s3_bucket,
Key=json_key,
Filename=local_model_metadata_path)
sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata(
local_model_metadata_path)
simapp_versions.append(simapp_version)
if Input.STEREO.value in sensors:
racecars_with_stereo_cameras.append(racecar_name)
if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors:
racecars_with_lidars.append(racecar_name)
cmd = [
''.join(("roslaunch deepracer_simulation_environment {} ".format(
launch_name), "local_yaml_path:={} ".format(local_yaml_path),
"racecars_with_stereo_cameras:={} ".format(
','.join(racecars_with_stereo_cameras)),
"racecars_with_lidars:={} multicar:={} ".format(
','.join(racecars_with_lidars), multicar),
"car_colors:={} simapp_versions:={}".format(
','.join(yaml_values[CAR_COLOR_YAML_KEY]),
','.join(simapp_versions))))
]
Popen(cmd, shell=True, executable="/bin/bash")
except botocore.exceptions.ClientError as ex:
log_and_exit(
"Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}"
.format(s3_bucket, yaml_key, ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400)
except botocore.exceptions.EndpointConnectionError:
log_and_exit("No Internet connection or s3 service unavailable",
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
except Exception as ex:
log_and_exit(
"Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}"
.format(s3_bucket, yaml_key, ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def main():
""" Main function for evaluation worker """
parser = argparse.ArgumentParser()
parser.add_argument('-p',
'--preset',
help="(string) Name of a preset to run \
(class name from the 'presets' directory.)",
type=str,
required=False)
parser.add_argument('--s3_bucket',
help='list(string) S3 bucket',
type=str,
nargs='+',
default=rospy.get_param("MODEL_S3_BUCKET",
["gsaur-test"]))
parser.add_argument('--s3_prefix',
help='list(string) S3 prefix',
type=str,
nargs='+',
default=rospy.get_param("MODEL_S3_PREFIX",
["sagemaker"]))
parser.add_argument('--s3_endpoint_url',
help='(string) S3 endpoint URL',
type=str,
default=rospy.get_param("S3_ENDPOINT_URL", None))
parser.add_argument('--aws_region',
help='(string) AWS region',
type=str,
default=rospy.get_param("AWS_REGION", "us-east-1"))
parser.add_argument('--number_of_trials',
help='(integer) Number of trials',
type=int,
default=int(rospy.get_param("NUMBER_OF_TRIALS", 10)))
parser.add_argument(
'-c',
'--local_model_directory',
help='(string) Path to a folder containing a checkpoint \
to restore the model from.',
type=str,
default='./checkpoint')
parser.add_argument('--number_of_resets',
help='(integer) Number of resets',
type=int,
default=int(rospy.get_param("NUMBER_OF_RESETS", 0)))
parser.add_argument('--penalty_seconds',
help='(float) penalty second',
type=float,
default=float(rospy.get_param("PENALTY_SECONDS", 2.0)))
parser.add_argument('--job_type',
help='(string) job type',
type=str,
default=rospy.get_param("JOB_TYPE", "EVALUATION"))
parser.add_argument('--is_continuous',
help='(boolean) is continous after lap completion',
type=bool,
default=utils.str2bool(
rospy.get_param("IS_CONTINUOUS", False)))
parser.add_argument('--race_type',
help='(string) Race type',
type=str,
default=rospy.get_param("RACE_TYPE", "TIME_TRIAL"))
parser.add_argument('--off_track_penalty',
help='(float) off track penalty second',
type=float,
default=float(rospy.get_param("OFF_TRACK_PENALTY",
2.0)))
parser.add_argument('--collision_penalty',
help='(float) collision penalty second',
type=float,
default=float(rospy.get_param("COLLISION_PENALTY",
5.0)))
parser.add_argument('--round_robin_advance_dist',
help='(float) round robin distance 0-1',
type=float,
default=float(
rospy.get_param("ROUND_ROBIN_ADVANCE_DIST", 0.05)))
parser.add_argument('--start_position_offset',
help='(float) offset start 0-1',
type=float,
default=float(
rospy.get_param("START_POSITION_OFFSET", 0.0)))
args = parser.parse_args()
arg_s3_bucket = args.s3_bucket
arg_s3_prefix = args.s3_prefix
logger.info("S3 bucket: %s \n S3 prefix: %s \n S3 endpoint URL: %s",
args.s3_bucket, args.s3_prefix, args.s3_endpoint_url)
metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET')
metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY')
arg_s3_bucket, arg_s3_prefix = utils.force_list(
arg_s3_bucket), utils.force_list(arg_s3_prefix)
metrics_s3_buckets = utils.force_list(metrics_s3_buckets)
metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys)
validate_list = [
arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets,
metrics_s3_object_keys
]
simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None)
mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None)
if simtrace_s3_bucket:
simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX')
simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket)
simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix)
validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix])
if mp4_s3_bucket:
mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX')
mp4_s3_bucket = utils.force_list(mp4_s3_bucket)
mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix)
validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix])
if not all([lambda x: len(x) == len(validate_list[0]), validate_list]):
log_and_exit(
"Eval worker error: Incorrect arguments passed: {}".format(
validate_list), SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT:
raise GenericRolloutException(
"number of resets is less than {}".format(MIN_RESET_COUNT))
# Instantiate Cameras
if len(arg_s3_bucket) == 1:
configure_camera(namespaces=['racecar'])
else:
configure_camera(namespaces=[
'racecar_{}'.format(str(agent_index))
for agent_index in range(len(arg_s3_bucket))
])
agent_list = list()
s3_bucket_dict = dict()
s3_prefix_dict = dict()
s3_writers = list()
start_positions = get_start_positions(len(arg_s3_bucket))
done_condition = utils.str_to_done_condition(
rospy.get_param("DONE_CONDITION", any))
park_positions = utils.pos_2d_str_to_list(
rospy.get_param("PARK_POSITIONS", []))
# if not pass in park positions for all done condition case, use default
if not park_positions:
park_positions = [DEFAULT_PARK_POSITION for _ in arg_s3_bucket]
for agent_index, _ in enumerate(arg_s3_bucket):
agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format(
str(agent_index))
racecar_name = 'racecar' if len(
arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index))
s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index]
s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index]
# download model metadata
model_metadata = ModelMetadata(
bucket=arg_s3_bucket[agent_index],
s3_key=get_s3_key(arg_s3_prefix[agent_index],
MODEL_METADATA_S3_POSTFIX),
region_name=args.aws_region,
s3_endpoint_url=args.s3_endpoint_url,
local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format(agent_name))
_, _, version = model_metadata.get_model_metadata_info()
# Select the optimal model
utils.do_model_selection(s3_bucket=arg_s3_bucket[agent_index],
s3_prefix=arg_s3_prefix[agent_index],
region=args.aws_region,
s3_endpoint_url=args.s3_endpoint_url)
agent_config = {
'model_metadata': model_metadata,
ConfigParams.CAR_CTRL_CONFIG.value: {
ConfigParams.LINK_NAME_LIST.value: [
link_name.replace('racecar', racecar_name)
for link_name in LINK_NAMES
],
ConfigParams.VELOCITY_LIST.value: [
velocity_topic.replace('racecar', racecar_name)
for velocity_topic in VELOCITY_TOPICS
],
ConfigParams.STEERING_LIST.value: [
steering_topic.replace('racecar', racecar_name)
for steering_topic in STEERING_TOPICS
],
ConfigParams.CHANGE_START.value:
utils.str2bool(rospy.get_param('CHANGE_START_POSITION',
False)),
ConfigParams.ALT_DIR.value:
utils.str2bool(
rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)),
ConfigParams.ACTION_SPACE_PATH.value:
model_metadata.local_path,
ConfigParams.REWARD.value:
reward_function,
ConfigParams.AGENT_NAME.value:
racecar_name,
ConfigParams.VERSION.value:
version,
ConfigParams.NUMBER_OF_RESETS.value:
args.number_of_resets,
ConfigParams.PENALTY_SECONDS.value:
args.penalty_seconds,
ConfigParams.NUMBER_OF_TRIALS.value:
args.number_of_trials,
ConfigParams.IS_CONTINUOUS.value:
args.is_continuous,
ConfigParams.RACE_TYPE.value:
args.race_type,
ConfigParams.COLLISION_PENALTY.value:
args.collision_penalty,
ConfigParams.OFF_TRACK_PENALTY.value:
args.off_track_penalty,
ConfigParams.START_POSITION.value:
start_positions[agent_index],
ConfigParams.DONE_CONDITION.value:
done_condition,
ConfigParams.ROUND_ROBIN_ADVANCE_DIST.value:
args.round_robin_advance_dist,
ConfigParams.START_POSITION_OFFSET.value:
args.start_position_offset
}
}
metrics_s3_config = {
MetricsS3Keys.METRICS_BUCKET.value:
metrics_s3_buckets[agent_index],
MetricsS3Keys.METRICS_KEY.value:
metrics_s3_object_keys[agent_index],
MetricsS3Keys.ENDPOINT_URL.value:
rospy.get_param('S3_ENDPOINT_URL', None),
# Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed
# or default argument set
MetricsS3Keys.REGION.value:
args.aws_region
}
aws_region = rospy.get_param('AWS_REGION', args.aws_region)
s3_writer_job_info = []
if simtrace_s3_bucket:
s3_writer_job_info.append(
IterationData(
'simtrace', simtrace_s3_bucket[agent_index],
simtrace_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
SIM_TRACE_EVALUATION_LOCAL_FILE.value)))
if mp4_s3_bucket:
s3_writer_job_info.extend([
IterationData(
'pip', mp4_s3_bucket[agent_index],
mp4_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
CAMERA_PIP_MP4_VALIDATION_LOCAL_PATH.value)),
IterationData(
'45degree', mp4_s3_bucket[agent_index],
mp4_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
CAMERA_45DEGREE_MP4_VALIDATION_LOCAL_PATH.value)),
IterationData(
'topview', mp4_s3_bucket[agent_index],
mp4_s3_object_prefix[agent_index], aws_region,
os.path.join(
ITERATION_DATA_LOCAL_FILE_PATH, agent_name,
IterationDataLocalFileNames.
CAMERA_TOPVIEW_MP4_VALIDATION_LOCAL_PATH.value))
])
s3_writers.append(
S3Writer(job_info=s3_writer_job_info,
s3_endpoint_url=args.s3_endpoint_url))
run_phase_subject = RunPhaseSubject()
agent_list.append(
create_rollout_agent(
agent_config,
EvalMetrics(agent_name, metrics_s3_config, args.is_continuous),
run_phase_subject))
agent_list.append(create_obstacles_agent())
agent_list.append(create_bot_cars_agent())
# ROS service to indicate all the robomaker markov packages are ready for consumption
signal_robomaker_markov_package_ready()
PhaseObserver('/agent/training_phase', run_phase_subject)
enable_domain_randomization = utils.str2bool(
rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False))
sm_hyperparams_dict = {}
graph_manager, _ = get_graph_manager(
hp_dict=sm_hyperparams_dict,
agent_list=agent_list,
run_phase_subject=run_phase_subject,
enable_domain_randomization=enable_domain_randomization,
done_condition=done_condition)
ds_params_instance = S3BotoDataStoreParameters(
aws_region=args.aws_region,
bucket_names=s3_bucket_dict,
base_checkpoint_dir=args.local_model_directory,
s3_folders=s3_prefix_dict,
s3_endpoint_url=args.s3_endpoint_url)
graph_manager.data_store = S3BotoDataStore(params=ds_params_instance,
graph_manager=graph_manager,
ignore_lock=True)
graph_manager.env_params.seed = 0
task_parameters = TaskParameters()
task_parameters.checkpoint_restore_path = args.local_model_directory
evaluation_worker(graph_manager=graph_manager,
number_of_trials=args.number_of_trials,
task_parameters=task_parameters,
s3_writers=s3_writers,
is_continuous=args.is_continuous,
park_positions=park_positions)
def __init__(self, racecar_names):
''' Constructor for the Deep Racer object, will load track and waypoints
'''
# Wait for required services to be available
rospy.wait_for_service(SET_MODEL_STATE)
rospy.wait_for_service(GazeboServiceName.PAUSE_PHYSICS.value)
rospy.wait_for_service(GazeboServiceName.GET_MODEL_PROPERTIES.value)
rospy.wait_for_service(GazeboServiceName.GET_VISUAL_NAMES.value)
rospy.wait_for_service(GazeboServiceName.GET_VISUALS.value)
rospy.wait_for_service(GazeboServiceName.SET_VISUAL_COLORS.value)
rospy.wait_for_service(GazeboServiceName.SET_VISUAL_TRANSPARENCIES.value)
rospy.wait_for_service(GazeboServiceName.SET_VISUAL_VISIBLES.value)
self.racer_num = len(racecar_names)
for racecar_name in racecar_names:
wait_for_model(model_name=racecar_name, relative_entity_name='')
self.car_colors = force_list(rospy.get_param(const.YamlKey.CAR_COLOR.value,
[const.CarColorType.BLACK.value] * len(racecar_names)))
self.shell_types = force_list(rospy.get_param(const.YamlKey.BODY_SHELL_TYPE.value,
[const.BodyShellType.DEFAULT.value] * len(racecar_names)))
# Gazebo service that allows us to position the car
self.model_state_client = ServiceProxyWrapper(SET_MODEL_STATE, SetModelState)
self.get_model_prop = ServiceProxyWrapper(GazeboServiceName.GET_MODEL_PROPERTIES.value,
GetModelProperties)
self.get_visual_names = ServiceProxyWrapper(GazeboServiceName.GET_VISUAL_NAMES.value,
GetVisualNames)
self.get_visuals = ServiceProxyWrapper(GazeboServiceName.GET_VISUALS.value, GetVisuals)
self.set_visual_colors = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_COLORS.value,
SetVisualColors)
self.set_visual_transparencies = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_TRANSPARENCIES.value,
SetVisualTransparencies)
self.set_visual_visibles = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_VISIBLES.value,
SetVisualVisibles)
# Place the car at the starting point facing the forward direction
# Instantiate cameras
main_cameras, sub_camera = configure_camera(namespaces=racecar_names)
[camera.detach() for camera in main_cameras.values()]
sub_camera.detach()
# Get the root directory of the ros package, this will contain the models
deepracer_path = rospkg.RosPack().get_path("deepracer_simulation_environment")
# Grab the track data
self.track_data = TrackData.get_instance()
# Set all racers start position in track data
self.start_positions = get_start_positions(self.racer_num)
car_poses = []
for racecar_idx, racecar_name in enumerate(racecar_names):
car_model_state = self.get_initial_position(racecar_name,
racecar_idx)
car_poses.append(car_model_state.pose)
self.update_model_visual(racecar_name,
self.shell_types[racecar_idx],
self.car_colors[racecar_idx])
# Let KVS collect a few frames before pausing the physics, so the car
# will appear on the track
time.sleep(1)
pause_physics = ServiceProxyWrapper('/gazebo/pause_physics', Empty)
logger.info("Pausing physics after initializing the cars")
pause_physics(EmptyRequest())
for racecar_name, car_pose in zip(racecar_names, car_poses):
main_cameras[racecar_name].spawn_model(car_pose,
os.path.join(deepracer_path, "models",
"camera", "model.sdf"))
logger.info("Spawning sub camera model")
# Spawn the top camera model
sub_camera.spawn_model(None, os.path.join(deepracer_path, "models",
"top_camera", "model.sdf"))
def main():
""" Main function for downloading yaml params """
# parse argument
s3_region = sys.argv[1]
s3_bucket = sys.argv[2]
s3_prefix = sys.argv[3]
s3_yaml_name = sys.argv[4]
launch_name = sys.argv[5]
yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name))
try:
# create boto3 session/client and download yaml/json file
session = boto3.session.Session()
s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None)
if s3_endpoint_url is not None:
LOG.info('Endpoint URL {}'.format(s3_endpoint_url))
rospy.set_param('S3_ENDPOINT_URL', s3_endpoint_url)
s3_client = session.client('s3',
region_name=s3_region,
endpoint_url=s3_endpoint_url,
config=get_boto_config())
local_yaml_path = os.path.abspath(
os.path.join(os.getcwd(), s3_yaml_name))
s3_client.download_file(Bucket=s3_bucket,
Key=yaml_key,
Filename=local_yaml_path)
# Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing
default_yaml_values = {
RACE_TYPE_YAML_KEY: TIME_TRIAL_RACE_TYPE,
MODEL_S3_BUCKET_YAML_KEY: s3_bucket,
MODEL_S3_PREFIX_YAML_KEY: s3_prefix,
CAR_COLOR_YAML_KEY: DEFAULT_COLOR,
BODY_SHELL_TYPE_YAML_KEY: None,
MODEL_METADATA_FILE_S3_YAML_KEY: None,
RACER_NAME_YAML_KEY: None
}
yaml_dict = get_yaml_dict(local_yaml_path)
yaml_values = get_yaml_values(yaml_dict, default_yaml_values)
# Forcing the yaml parameter to list
force_list_params = [
MODEL_METADATA_FILE_S3_YAML_KEY, MODEL_S3_BUCKET_YAML_KEY,
MODEL_S3_PREFIX_YAML_KEY, CAR_COLOR_YAML_KEY,
BODY_SHELL_TYPE_YAML_KEY, RACER_NAME_YAML_KEY
]
for params in force_list_params:
yaml_values[params] = force_list(yaml_values[params])
# Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats
if None in yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY]:
# MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation
# Assume model_metadata.json is present in the s3_prefix/model/ folder
yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY] = list()
for s3_prefix in yaml_values[MODEL_S3_PREFIX_YAML_KEY]:
yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY].append(
os.path.join(s3_prefix, 'model/model_metadata.json'))
# Set multicar value if there is more than one value in MODEL_S3_BUCKET_YAML_KEY.
multicar = len(yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1
# Set f1 as true if RACE_TYPE is F1
is_f1 = yaml_values[RACE_TYPE_YAML_KEY] == F1_RACE_TYPE
# Validate the yaml values
validate_yaml_values(yaml_values, multicar)
# List of racecar names that should include second camera while launching
racecars_with_stereo_cameras = list()
# List of racecar names that should include lidar while launching
racecars_with_lidars = list()
# List of SimApp versions
simapp_versions = list()
# List of body shell types
body_shell_types = yaml_values[BODY_SHELL_TYPE_YAML_KEY]
racer_names = yaml_values[RACER_NAME_YAML_KEY]
if None in body_shell_types:
# use default shells
if None in racer_names:
body_shell_types = [BodyShellType.DEFAULT.value] * len(
yaml_values[MODEL_S3_BUCKET_YAML_KEY])
# use default shells for regular user and f1 shell for users in F1_SHELL_USERS_LIST
else:
body_shell_types = [
BodyShellType.F1_2021.value if racer_alias
in F1_SHELL_USERS_LIST else BodyShellType.DEFAULT.value
for racer_alias in yaml_values[RACER_NAME_YAML_KEY]
]
yaml_dict[BODY_SHELL_TYPE_YAML_KEY] = body_shell_types
# override local yaml file with updated BODY_SHELL_TYPE
with open(local_yaml_path, 'w') as yaml_file:
yaml.dump(yaml_dict, yaml_file)
for agent_index, model_s3_bucket in enumerate(
yaml_values[MODEL_S3_BUCKET_YAML_KEY]):
racecar_name = 'racecar_' + str(agent_index) if len(
yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 else 'racecar'
# Make a local folder with the racecar name to download the model_metadata.json
if not os.path.exists(os.path.join(os.getcwd(), racecar_name)):
os.makedirs(os.path.join(os.getcwd(), racecar_name))
local_model_metadata_path = os.path.abspath(
os.path.join(os.path.join(os.getcwd(), racecar_name),
'model_metadata.json'))
json_key = yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY][
agent_index]
json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '')
s3_client.download_file(Bucket=model_s3_bucket,
Key=json_key,
Filename=local_model_metadata_path)
sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata(
local_model_metadata_path)
simapp_versions.append(simapp_version)
if Input.STEREO.value in sensors:
racecars_with_stereo_cameras.append(racecar_name)
if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors:
racecars_with_lidars.append(racecar_name)
cmd = [
''.join(("roslaunch deepracer_simulation_environment {} ".format(
launch_name), "local_yaml_path:={} ".format(local_yaml_path),
"racecars_with_stereo_cameras:={} ".format(
','.join(racecars_with_stereo_cameras)),
"racecars_with_lidars:={} ".format(
','.join(racecars_with_lidars)),
"multicar:={} ".format(multicar),
"body_shell_types:={} ".format(
','.join(body_shell_types)),
"simapp_versions:={} ".format(','.join(simapp_versions)),
"f1:={}".format(is_f1)))
]
Popen(cmd, shell=True, executable="/bin/bash")
except botocore.exceptions.ClientError as ex:
log_and_exit(
"Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}"
.format(s3_bucket, yaml_key, ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400)
except botocore.exceptions.EndpointConnectionError:
log_and_exit("No Internet connection or s3 service unavailable",
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
except Exception as ex:
log_and_exit(
"Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}"
.format(s3_bucket, yaml_key, ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
