def _spawn_cameras(self): '''helper method for initializing cameras ''' # virtual event configure camera does not need to wait for car to spawm because # follow car camera is not tracking any car initially camera_manager = CameraManager.get_instance() # pop all camera under virtual event namespace camera_manager.pop(namespace=VIRTUAL_EVENT) # Spawn the follow car camera LOG.info( "[virtual event manager] Spawning virtual event follow car camera model" ) initial_pose = self._track_data.get_racecar_start_pose( racecar_idx=0, racer_num=1, start_position=get_start_positions(1)[0]) self._main_cameras[VIRTUAL_EVENT].spawn_model( initial_pose, os.path.join(self._deepracer_path, "models", "camera", "model.sdf")) LOG.info("[virtual event manager] Spawning sub camera model") # Spawn the top camera model self._sub_camera.spawn_model( None, os.path.join(self._deepracer_path, "models", "top_camera", "model.sdf"))
def _get_agent_list(self, model_metadata, version): """Setup agent and get the agents list. Args: model_metadata (ModelMetadata): Current racer's model metadata version (str): The current racer's simapp version in the model metadata Returns: agent_list (list): The list of agents for the current racer """ # setup agent agent_config = { "model_metadata": model_metadata, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [ link_name.replace("racecar", self._current_car_model_state.model_name) for link_name in LINK_NAMES ], ConfigParams.VELOCITY_LIST.value: [ velocity_topic.replace("racecar", self._current_car_model_state.model_name) for velocity_topic in VELOCITY_TOPICS ], ConfigParams.STEERING_LIST.value: [ steering_topic.replace("racecar", self._current_car_model_state.model_name) for steering_topic in STEERING_TOPICS ], ConfigParams.CHANGE_START.value: utils.str2bool( rospy.get_param("CHANGE_START_POSITION", False) ), ConfigParams.ALT_DIR.value: utils.str2bool( rospy.get_param("ALTERNATE_DRIVING_DIRECTION", False) ), ConfigParams.MODEL_METADATA.value: model_metadata, ConfigParams.REWARD.value: reward_function, ConfigParams.AGENT_NAME.value: self._current_car_model_state.model_name, ConfigParams.VERSION.value: version, ConfigParams.NUMBER_OF_RESETS.value: self._number_of_resets, ConfigParams.PENALTY_SECONDS.value: self._penalty_seconds, ConfigParams.NUMBER_OF_TRIALS.value: self._number_of_trials, ConfigParams.IS_CONTINUOUS.value: self._is_continuous, ConfigParams.RACE_TYPE.value: self._race_type, ConfigParams.COLLISION_PENALTY.value: self._collision_penalty, ConfigParams.OFF_TRACK_PENALTY.value: self._off_track_penalty, ConfigParams.START_POSITION.value: get_start_positions(1)[ 0 ], # hard-coded to the first start position ConfigParams.DONE_CONDITION.value: self._done_condition, ConfigParams.IS_VIRTUAL_EVENT.value: True, ConfigParams.RACE_DURATION.value: self._race_duration, }, } agent_list = list() agent_list.append( create_rollout_agent(agent_config, self._eval_metrics, self._run_phase_subject) ) agent_list.append(create_obstacles_agent()) agent_list.append(create_bot_cars_agent()) return agent_list
def setup_race(self): """ Setting up the race for the current racer. Returns: bool: True if setup race is successful. False is a non fatal exception occurred. """ LOG.info("[virtual event manager] Setting up race for racer") try: # unpause the physics in current world self._model_updater.unpause_physics() LOG.info("[virtual event manager] Unpaused physics in current world.") # set camera to starting position initial_pose = self._track_data.get_racecar_start_pose( racecar_idx=0, racer_num=1, start_position=get_start_positions(1)[0]) self._main_cameras[VIRTUAL_EVENT].reset_pose( car_pose=initial_pose) LOG.info("[virtual event manager] Reset camera to starting line.") if self._prev_model_name is not None: # NOTE: it's by design that we immediately part the previous car to pit # location right after unpause physics. This prevents any unwanted # leftover behavior to happen self._park_at_pit_location(self._prev_model_name) LOG.info("[virtual event manager] Parked previous model %s to pit location.", self._prev_model_name) self._model_updater.pause_physics() LOG.info("[virtual event manager] Paused physics in current world.") # download model metadata from s3 sensors, version, model_metadata = self._download_model_metadata() # based on model metadata, select racecar self._current_car_model_state = self._get_car_model_state(sensors) # download checkpoint from s3 checkpoint = self._download_checkpoint(version) # setup the simtrace and mp4 writers if the s3 locations are available self._setup_simtrace_mp4_writers() # reset the metrics s3 location for the current racer self._reset_metrics_loc() # setup agents agent_list = self._get_agent_list(model_metadata, version) self._setup_graph_manager(checkpoint, agent_list) LOG.info("[virtual event manager] Graph manager successfully created the graph: setup race successful.") return True except GenericNonFatalException as ex: ex.log_except_and_continue() self.upload_race_status(status_code=ex.error_code, error_name=ex.error_name, error_details=ex.error_msg) self._clean_up_race() return False except Exception as ex: log_and_exit("[virtual event manager] Something really wrong happened when setting up the race. {}".format(ex), SIMAPP_VIRTUAL_EVENT_RACE_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def setup_race(self): """ Setting up the race for the current racer. Returns: bool: True if setup race is successful. False is a non fatal exception occurred. """ LOG.info("[virtual event manager] Setting up race for racer") try: self._model_updater.unpause_physics() LOG.info( "[virtual event manager] Unpause physics in current world to setup race." ) # step 1: hide the racecar to a position that camera cannot see self._hide_racecar_model( model_name=self._current_car_model_state.model_name) # step 2: set camera to starting position after previous car is deleted initial_pose = self._track_data.get_racecar_start_pose( racecar_idx=0, racer_num=1, start_position=get_start_positions(1)[0]) self._main_cameras[VIRTUAL_EVENT].reset_pose(car_pose=initial_pose) LOG.info("[virtual event manager] Reset camera to starting line.") # step 3: download model metadata from s3 sensors, version, model_metadata = self._download_model_metadata() # step 4: check whether body shell and sensors have been updated # to decide whether need to delete and re-spawn. Then, update # shell or color accordingly if hasattr(self._current_racer, "carConfig") and \ hasattr(self._current_racer.carConfig, "bodyShellType"): body_shell_type = self._current_racer.carConfig.bodyShellType \ if self._current_racer.carConfig.bodyShellType in self._valid_body_shells \ else const.BodyShellType.DEFAULT.value else: body_shell_type = const.BodyShellType.DEFAULT.value # check whether need to delete and respawn racecar # re-spawn if sensor or body shell type changed if self._last_body_shell_type != body_shell_type or \ self._last_sensors != sensors: # delete last racecar self._racecar_model.delete() # respawn a new racecar hide_pose = Pose() hide_pose.position.x = self._hide_positions[ self._hide_position_idx][0] hide_pose.position.y = self._hide_positions[ self._hide_position_idx][1] self._racecar_model.spawn( name=self._current_car_model_state.model_name, pose=hide_pose, include_second_camera="true" if Input.STEREO.value in sensors else "false", include_lidar_sensor=str( any(["lidar" in sensor.lower() for sensor in sensors])).lower(), body_shell_type=body_shell_type, lidar_360_degree_sample=str(LIDAR_360_DEGREE_SAMPLE), lidar_360_degree_horizontal_resolution=str( LIDAR_360_DEGREE_HORIZONTAL_RESOLUTION), lidar_360_degree_min_angle=str(LIDAR_360_DEGREE_MIN_ANGLE), lidar_360_degree_max_angle=str(LIDAR_360_DEGREE_MAX_ANGLE), lidar_360_degree_min_range=str(LIDAR_360_DEGREE_MIN_RANGE), lidar_360_degree_max_range=str(LIDAR_360_DEGREE_MAX_RANGE), lidar_360_degree_range_resolution=str( LIDAR_360_DEGREE_RANGE_RESOLUTION), lidar_360_degree_noise_mean=str( LIDAR_360_DEGREE_NOISE_MEAN), lidar_360_degree_noise_stddev=str( LIDAR_360_DEGREE_NOISE_STDDEV)) self._last_body_shell_type = body_shell_type self._last_sensors = sensors # step 5: download checkpoint, setup simtrace, mp4, clear metrics, and setup graph manager # download checkpoint from s3 checkpoint = self._download_checkpoint(version) # setup the simtrace and mp4 writers if the s3 locations are available self._setup_simtrace_mp4_writers() # reset the metrics s3 location for the current racer self._reset_metrics_loc() # setup agents agent_list = self._get_agent_list(model_metadata, version) # after _setup_graph_manager finishes, physics is paused # physics will be unpaused again when race start self._setup_graph_manager(checkpoint, agent_list) LOG.info( "[virtual event manager] Graph manager successfully created the graph: setup race successful." ) # step 6: update body shell or color # treat amazon van digital reward specially by also hiding the collision wheel visuals = self._model_updater.get_model_visuals( self._current_car_model_state.model_name) if const.F1 in body_shell_type: self._model_updater.hide_visuals( visuals=visuals, ignore_keywords=["f1_body_link"] if "with_wheel" in body_shell_type.lower() else ["wheel", "f1_body_link"]) else: if hasattr(self._current_racer, "carConfig") and \ hasattr(self._current_racer.carConfig, "carColor"): car_color = self._current_racer.carConfig.carColor if self._current_racer.carConfig.carColor in self._valid_car_colors \ else DEFAULT_COLOR else: car_color = DEFAULT_COLOR self._model_updater.update_color(visuals, car_color) return True except GenericNonFatalException as ex: ex.log_except_and_continue() self.upload_race_status(status_code=ex.error_code, error_name=ex.error_name, error_details=ex.error_msg) self._clean_up_race() return False except Exception as ex: log_and_exit( "[virtual event manager] Something really wrong happened when setting up the race. {}" .format(ex), SIMAPP_VIRTUAL_EVENT_RACE_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def main(): """ Main function for evaluation worker """ parser = argparse.ArgumentParser() parser.add_argument('-p', '--preset', help="(string) Name of a preset to run \ (class name from the 'presets' directory.)", type=str, required=False) parser.add_argument('--s3_bucket', help='list(string) S3 bucket', type=str, nargs='+', default=rospy.get_param("MODEL_S3_BUCKET", ["gsaur-test"])) parser.add_argument('--s3_prefix', help='list(string) S3 prefix', type=str, nargs='+', default=rospy.get_param("MODEL_S3_PREFIX", ["sagemaker"])) parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=rospy.get_param("AWS_REGION", "us-east-1")) parser.add_argument('--number_of_trials', help='(integer) Number of trials', type=int, default=int(rospy.get_param("NUMBER_OF_TRIALS", 10))) parser.add_argument( '-c', '--local_model_directory', help='(string) Path to a folder containing a checkpoint \ to restore the model from.', type=str, default='./checkpoint') parser.add_argument('--number_of_resets', help='(integer) Number of resets', type=int, default=int(rospy.get_param("NUMBER_OF_RESETS", 0))) parser.add_argument('--penalty_seconds', help='(float) penalty second', type=float, default=float(rospy.get_param("PENALTY_SECONDS", 2.0))) parser.add_argument('--job_type', help='(string) job type', type=str, default=rospy.get_param("JOB_TYPE", "EVALUATION")) parser.add_argument('--is_continuous', help='(boolean) is continous after lap completion', type=bool, default=utils.str2bool( rospy.get_param("IS_CONTINUOUS", False))) parser.add_argument('--race_type', help='(string) Race type', type=str, default=rospy.get_param("RACE_TYPE", "TIME_TRIAL")) parser.add_argument('--off_track_penalty', help='(float) off track penalty second', type=float, default=float(rospy.get_param("OFF_TRACK_PENALTY", 2.0))) parser.add_argument('--collision_penalty', help='(float) collision penalty second', type=float, default=float(rospy.get_param("COLLISION_PENALTY", 5.0))) args = parser.parse_args() arg_s3_bucket = args.s3_bucket arg_s3_prefix = args.s3_prefix logger.info("S3 bucket: %s \n S3 prefix: %s", arg_s3_bucket, arg_s3_prefix) metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET') metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY') arg_s3_bucket, arg_s3_prefix = utils.force_list( arg_s3_bucket), utils.force_list(arg_s3_prefix) metrics_s3_buckets = utils.force_list(metrics_s3_buckets) metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys) validate_list = [ arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets, metrics_s3_object_keys ] simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None) mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None) if simtrace_s3_bucket: simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX') simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket) simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix) validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix]) if mp4_s3_bucket: mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX') mp4_s3_bucket = utils.force_list(mp4_s3_bucket) mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix) validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix]) if not all([lambda x: len(x) == len(validate_list[0]), validate_list]): log_and_exit( "Eval worker error: Incorrect arguments passed: {}".format( validate_list), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT: raise GenericRolloutException( "number of resets is less than {}".format(MIN_RESET_COUNT)) # Instantiate Cameras if len(arg_s3_bucket) == 1: configure_camera(namespaces=['racecar']) else: configure_camera(namespaces=[ 'racecar_{}'.format(str(agent_index)) for agent_index in range(len(arg_s3_bucket)) ]) agent_list = list() s3_bucket_dict = dict() s3_prefix_dict = dict() checkpoint_dict = dict() simtrace_video_s3_writers = [] start_positions = get_start_positions(len(arg_s3_bucket)) done_condition = utils.str_to_done_condition( rospy.get_param("DONE_CONDITION", any)) park_positions = utils.pos_2d_str_to_list( rospy.get_param("PARK_POSITIONS", [])) # if not pass in park positions for all done condition case, use default if not park_positions: park_positions = [DEFAULT_PARK_POSITION for _ in arg_s3_bucket] for agent_index, _ in enumerate(arg_s3_bucket): agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format( str(agent_index)) racecar_name = 'racecar' if len( arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index)) s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index] s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index] # download model metadata model_metadata = ModelMetadata( bucket=arg_s3_bucket[agent_index], s3_key=get_s3_key(arg_s3_prefix[agent_index], MODEL_METADATA_S3_POSTFIX), region_name=args.aws_region, local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format(agent_name)) model_metadata_info = model_metadata.get_model_metadata_info() version = model_metadata_info[ModelMetadataKeys.VERSION.value] # checkpoint s3 instance checkpoint = Checkpoint(bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], region_name=args.aws_region, agent_name=agent_name, checkpoint_dir=args.local_model_directory) # make coach checkpoint compatible if version < SIMAPP_VERSION_2 and not checkpoint.rl_coach_checkpoint.is_compatible( ): checkpoint.rl_coach_checkpoint.make_compatible( checkpoint.syncfile_ready) # get best model checkpoint string model_checkpoint_name = checkpoint.deepracer_checkpoint_json.get_deepracer_best_checkpoint( ) # Select the best checkpoint model by uploading rl coach .coach_checkpoint file checkpoint.rl_coach_checkpoint.update( model_checkpoint_name=model_checkpoint_name, s3_kms_extra_args=utils.get_s3_kms_extra_args()) checkpoint_dict[agent_name] = checkpoint agent_config = { 'model_metadata': model_metadata, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [ link_name.replace('racecar', racecar_name) for link_name in LINK_NAMES ], ConfigParams.VELOCITY_LIST.value: [ velocity_topic.replace('racecar', racecar_name) for velocity_topic in VELOCITY_TOPICS ], ConfigParams.STEERING_LIST.value: [ steering_topic.replace('racecar', racecar_name) for steering_topic in STEERING_TOPICS ], ConfigParams.CHANGE_START.value: utils.str2bool(rospy.get_param('CHANGE_START_POSITION', False)), ConfigParams.ALT_DIR.value: utils.str2bool( rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)), ConfigParams.MODEL_METADATA.value: model_metadata, ConfigParams.REWARD.value: reward_function, ConfigParams.AGENT_NAME.value: racecar_name, ConfigParams.VERSION.value: version, ConfigParams.NUMBER_OF_RESETS.value: args.number_of_resets, ConfigParams.PENALTY_SECONDS.value: args.penalty_seconds, ConfigParams.NUMBER_OF_TRIALS.value: args.number_of_trials, ConfigParams.IS_CONTINUOUS.value: args.is_continuous, ConfigParams.RACE_TYPE.value: args.race_type, ConfigParams.COLLISION_PENALTY.value: args.collision_penalty, ConfigParams.OFF_TRACK_PENALTY.value: args.off_track_penalty, ConfigParams.START_POSITION.value: start_positions[agent_index], ConfigParams.DONE_CONDITION.value: done_condition } } metrics_s3_config = { MetricsS3Keys.METRICS_BUCKET.value: metrics_s3_buckets[agent_index], MetricsS3Keys.METRICS_KEY.value: metrics_s3_object_keys[agent_index], # Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed # or default argument set MetricsS3Keys.REGION.value: args.aws_region } aws_region = rospy.get_param('AWS_REGION', args.aws_region) if simtrace_s3_bucket: simtrace_video_s3_writers.append( SimtraceVideo( upload_type=SimtraceVideoNames.SIMTRACE_EVAL.value, bucket=simtrace_s3_bucket[agent_index], s3_prefix=simtrace_s3_object_prefix[agent_index], region_name=aws_region, local_path=SIMTRACE_EVAL_LOCAL_PATH_FORMAT.format( agent_name))) if mp4_s3_bucket: simtrace_video_s3_writers.extend([ SimtraceVideo( upload_type=SimtraceVideoNames.PIP.value, bucket=mp4_s3_bucket[agent_index], s3_prefix=mp4_s3_object_prefix[agent_index], region_name=aws_region, local_path=CAMERA_PIP_MP4_LOCAL_PATH_FORMAT.format( agent_name)), SimtraceVideo( upload_type=SimtraceVideoNames.DEGREE45.value, bucket=mp4_s3_bucket[agent_index], s3_prefix=mp4_s3_object_prefix[agent_index], region_name=aws_region, local_path=CAMERA_45DEGREE_LOCAL_PATH_FORMAT.format( agent_name)), SimtraceVideo( upload_type=SimtraceVideoNames.TOPVIEW.value, bucket=mp4_s3_bucket[agent_index], s3_prefix=mp4_s3_object_prefix[agent_index], region_name=aws_region, local_path=CAMERA_TOPVIEW_LOCAL_PATH_FORMAT.format( agent_name)) ]) run_phase_subject = RunPhaseSubject() agent_list.append( create_rollout_agent( agent_config, EvalMetrics(agent_name, metrics_s3_config, args.is_continuous), run_phase_subject)) agent_list.append(create_obstacles_agent()) agent_list.append(create_bot_cars_agent()) # ROS service to indicate all the robomaker markov packages are ready for consumption signal_robomaker_markov_package_ready() PhaseObserver('/agent/training_phase', run_phase_subject) enable_domain_randomization = utils.str2bool( rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False)) sm_hyperparams_dict = {} # Make the clients that will allow us to pause and unpause the physics rospy.wait_for_service('/gazebo/pause_physics_dr') rospy.wait_for_service('/gazebo/unpause_physics_dr') pause_physics = ServiceProxyWrapper('/gazebo/pause_physics_dr', Empty) unpause_physics = ServiceProxyWrapper('/gazebo/unpause_physics_dr', Empty) graph_manager, _ = get_graph_manager( hp_dict=sm_hyperparams_dict, agent_list=agent_list, run_phase_subject=run_phase_subject, enable_domain_randomization=enable_domain_randomization, done_condition=done_condition, pause_physics=pause_physics, unpause_physics=unpause_physics) ds_params_instance = S3BotoDataStoreParameters( checkpoint_dict=checkpoint_dict) graph_manager.data_store = S3BotoDataStore(params=ds_params_instance, graph_manager=graph_manager, ignore_lock=True) graph_manager.env_params.seed = 0 task_parameters = TaskParameters() task_parameters.checkpoint_restore_path = args.local_model_directory evaluation_worker(graph_manager=graph_manager, number_of_trials=args.number_of_trials, task_parameters=task_parameters, simtrace_video_s3_writers=simtrace_video_s3_writers, is_continuous=args.is_continuous, park_positions=park_positions, race_type=args.race_type, pause_physics=pause_physics, unpause_physics=unpause_physics)
def main(): """ Main function for evaluation worker """ parser = argparse.ArgumentParser() parser.add_argument('-p', '--preset', help="(string) Name of a preset to run \ (class name from the 'presets' directory.)", type=str, required=False) parser.add_argument('--s3_bucket', help='list(string) S3 bucket', type=str, nargs='+', default=rospy.get_param("MODEL_S3_BUCKET", ["gsaur-test"])) parser.add_argument('--s3_prefix', help='list(string) S3 prefix', type=str, nargs='+', default=rospy.get_param("MODEL_S3_PREFIX", ["sagemaker"])) parser.add_argument('--s3_endpoint_url', help='(string) S3 endpoint URL', type=str, default=rospy.get_param("S3_ENDPOINT_URL", None)) parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=rospy.get_param("AWS_REGION", "us-east-1")) parser.add_argument('--number_of_trials', help='(integer) Number of trials', type=int, default=int(rospy.get_param("NUMBER_OF_TRIALS", 10))) parser.add_argument( '-c', '--local_model_directory', help='(string) Path to a folder containing a checkpoint \ to restore the model from.', type=str, default='./checkpoint') parser.add_argument('--number_of_resets', help='(integer) Number of resets', type=int, default=int(rospy.get_param("NUMBER_OF_RESETS", 0))) parser.add_argument('--penalty_seconds', help='(float) penalty second', type=float, default=float(rospy.get_param("PENALTY_SECONDS", 2.0))) parser.add_argument('--job_type', help='(string) job type', type=str, default=rospy.get_param("JOB_TYPE", "EVALUATION")) parser.add_argument('--is_continuous', help='(boolean) is continous after lap completion', type=bool, default=utils.str2bool( rospy.get_param("IS_CONTINUOUS", False))) parser.add_argument('--race_type', help='(string) Race type', type=str, default=rospy.get_param("RACE_TYPE", "TIME_TRIAL")) parser.add_argument('--off_track_penalty', help='(float) off track penalty second', type=float, default=float(rospy.get_param("OFF_TRACK_PENALTY", 2.0))) parser.add_argument('--collision_penalty', help='(float) collision penalty second', type=float, default=float(rospy.get_param("COLLISION_PENALTY", 5.0))) parser.add_argument('--round_robin_advance_dist', help='(float) round robin distance 0-1', type=float, default=float( rospy.get_param("ROUND_ROBIN_ADVANCE_DIST", 0.05))) parser.add_argument('--start_position_offset', help='(float) offset start 0-1', type=float, default=float( rospy.get_param("START_POSITION_OFFSET", 0.0))) args = parser.parse_args() arg_s3_bucket = args.s3_bucket arg_s3_prefix = args.s3_prefix logger.info("S3 bucket: %s \n S3 prefix: %s \n S3 endpoint URL: %s", args.s3_bucket, args.s3_prefix, args.s3_endpoint_url) metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET') metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY') arg_s3_bucket, arg_s3_prefix = utils.force_list( arg_s3_bucket), utils.force_list(arg_s3_prefix) metrics_s3_buckets = utils.force_list(metrics_s3_buckets) metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys) validate_list = [ arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets, metrics_s3_object_keys ] simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None) mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None) if simtrace_s3_bucket: simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX') simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket) simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix) validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix]) if mp4_s3_bucket: mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX') mp4_s3_bucket = utils.force_list(mp4_s3_bucket) mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix) validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix]) if not all([lambda x: len(x) == len(validate_list[0]), validate_list]): log_and_exit( "Eval worker error: Incorrect arguments passed: {}".format( validate_list), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT: raise GenericRolloutException( "number of resets is less than {}".format(MIN_RESET_COUNT)) # Instantiate Cameras if len(arg_s3_bucket) == 1: configure_camera(namespaces=['racecar']) else: configure_camera(namespaces=[ 'racecar_{}'.format(str(agent_index)) for agent_index in range(len(arg_s3_bucket)) ]) agent_list = list() s3_bucket_dict = dict() s3_prefix_dict = dict() s3_writers = list() start_positions = get_start_positions(len(arg_s3_bucket)) done_condition = utils.str_to_done_condition( rospy.get_param("DONE_CONDITION", any)) park_positions = utils.pos_2d_str_to_list( rospy.get_param("PARK_POSITIONS", [])) # if not pass in park positions for all done condition case, use default if not park_positions: park_positions = [DEFAULT_PARK_POSITION for _ in arg_s3_bucket] for agent_index, _ in enumerate(arg_s3_bucket): agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format( str(agent_index)) racecar_name = 'racecar' if len( arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index)) s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index] s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index] # download model metadata model_metadata = ModelMetadata( bucket=arg_s3_bucket[agent_index], s3_key=get_s3_key(arg_s3_prefix[agent_index], MODEL_METADATA_S3_POSTFIX), region_name=args.aws_region, s3_endpoint_url=args.s3_endpoint_url, local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format(agent_name)) _, _, version = model_metadata.get_model_metadata_info() # Select the optimal model utils.do_model_selection(s3_bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], region=args.aws_region, s3_endpoint_url=args.s3_endpoint_url) agent_config = { 'model_metadata': model_metadata, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [ link_name.replace('racecar', racecar_name) for link_name in LINK_NAMES ], ConfigParams.VELOCITY_LIST.value: [ velocity_topic.replace('racecar', racecar_name) for velocity_topic in VELOCITY_TOPICS ], ConfigParams.STEERING_LIST.value: [ steering_topic.replace('racecar', racecar_name) for steering_topic in STEERING_TOPICS ], ConfigParams.CHANGE_START.value: utils.str2bool(rospy.get_param('CHANGE_START_POSITION', False)), ConfigParams.ALT_DIR.value: utils.str2bool( rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)), ConfigParams.ACTION_SPACE_PATH.value: model_metadata.local_path, ConfigParams.REWARD.value: reward_function, ConfigParams.AGENT_NAME.value: racecar_name, ConfigParams.VERSION.value: version, ConfigParams.NUMBER_OF_RESETS.value: args.number_of_resets, ConfigParams.PENALTY_SECONDS.value: args.penalty_seconds, ConfigParams.NUMBER_OF_TRIALS.value: args.number_of_trials, ConfigParams.IS_CONTINUOUS.value: args.is_continuous, ConfigParams.RACE_TYPE.value: args.race_type, ConfigParams.COLLISION_PENALTY.value: args.collision_penalty, ConfigParams.OFF_TRACK_PENALTY.value: args.off_track_penalty, ConfigParams.START_POSITION.value: start_positions[agent_index], ConfigParams.DONE_CONDITION.value: done_condition, ConfigParams.ROUND_ROBIN_ADVANCE_DIST.value: args.round_robin_advance_dist, ConfigParams.START_POSITION_OFFSET.value: args.start_position_offset } } metrics_s3_config = { MetricsS3Keys.METRICS_BUCKET.value: metrics_s3_buckets[agent_index], MetricsS3Keys.METRICS_KEY.value: metrics_s3_object_keys[agent_index], MetricsS3Keys.ENDPOINT_URL.value: rospy.get_param('S3_ENDPOINT_URL', None), # Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed # or default argument set MetricsS3Keys.REGION.value: args.aws_region } aws_region = rospy.get_param('AWS_REGION', args.aws_region) s3_writer_job_info = [] if simtrace_s3_bucket: s3_writer_job_info.append( IterationData( 'simtrace', simtrace_s3_bucket[agent_index], simtrace_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. SIM_TRACE_EVALUATION_LOCAL_FILE.value))) if mp4_s3_bucket: s3_writer_job_info.extend([ IterationData( 'pip', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_PIP_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( '45degree', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_45DEGREE_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( 'topview', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_TOPVIEW_MP4_VALIDATION_LOCAL_PATH.value)) ]) s3_writers.append( S3Writer(job_info=s3_writer_job_info, s3_endpoint_url=args.s3_endpoint_url)) run_phase_subject = RunPhaseSubject() agent_list.append( create_rollout_agent( agent_config, EvalMetrics(agent_name, metrics_s3_config, args.is_continuous), run_phase_subject)) agent_list.append(create_obstacles_agent()) agent_list.append(create_bot_cars_agent()) # ROS service to indicate all the robomaker markov packages are ready for consumption signal_robomaker_markov_package_ready() PhaseObserver('/agent/training_phase', run_phase_subject) enable_domain_randomization = utils.str2bool( rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False)) sm_hyperparams_dict = {} graph_manager, _ = get_graph_manager( hp_dict=sm_hyperparams_dict, agent_list=agent_list, run_phase_subject=run_phase_subject, enable_domain_randomization=enable_domain_randomization, done_condition=done_condition) ds_params_instance = S3BotoDataStoreParameters( aws_region=args.aws_region, bucket_names=s3_bucket_dict, base_checkpoint_dir=args.local_model_directory, s3_folders=s3_prefix_dict, s3_endpoint_url=args.s3_endpoint_url) graph_manager.data_store = S3BotoDataStore(params=ds_params_instance, graph_manager=graph_manager, ignore_lock=True) graph_manager.env_params.seed = 0 task_parameters = TaskParameters() task_parameters.checkpoint_restore_path = args.local_model_directory evaluation_worker(graph_manager=graph_manager, number_of_trials=args.number_of_trials, task_parameters=task_parameters, s3_writers=s3_writers, is_continuous=args.is_continuous, park_positions=park_positions)
def __init__(self, racecar_names): ''' Constructor for the Deep Racer object, will load track and waypoints ''' # Wait for required services to be available rospy.wait_for_service(SET_MODEL_STATE) rospy.wait_for_service(GazeboServiceName.PAUSE_PHYSICS.value) rospy.wait_for_service(GazeboServiceName.GET_MODEL_PROPERTIES.value) rospy.wait_for_service(GazeboServiceName.GET_VISUAL_NAMES.value) rospy.wait_for_service(GazeboServiceName.GET_VISUALS.value) rospy.wait_for_service(GazeboServiceName.SET_VISUAL_COLORS.value) rospy.wait_for_service(GazeboServiceName.SET_VISUAL_TRANSPARENCIES.value) rospy.wait_for_service(GazeboServiceName.SET_VISUAL_VISIBLES.value) self.racer_num = len(racecar_names) for racecar_name in racecar_names: wait_for_model(model_name=racecar_name, relative_entity_name='') self.car_colors = force_list(rospy.get_param(const.YamlKey.CAR_COLOR.value, [const.CarColorType.BLACK.value] * len(racecar_names))) self.shell_types = force_list(rospy.get_param(const.YamlKey.BODY_SHELL_TYPE.value, [const.BodyShellType.DEFAULT.value] * len(racecar_names))) # Gazebo service that allows us to position the car self.model_state_client = ServiceProxyWrapper(SET_MODEL_STATE, SetModelState) self.get_model_prop = ServiceProxyWrapper(GazeboServiceName.GET_MODEL_PROPERTIES.value, GetModelProperties) self.get_visual_names = ServiceProxyWrapper(GazeboServiceName.GET_VISUAL_NAMES.value, GetVisualNames) self.get_visuals = ServiceProxyWrapper(GazeboServiceName.GET_VISUALS.value, GetVisuals) self.set_visual_colors = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_COLORS.value, SetVisualColors) self.set_visual_transparencies = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_TRANSPARENCIES.value, SetVisualTransparencies) self.set_visual_visibles = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_VISIBLES.value, SetVisualVisibles) # Place the car at the starting point facing the forward direction # Instantiate cameras main_cameras, sub_camera = configure_camera(namespaces=racecar_names) [camera.detach() for camera in main_cameras.values()] sub_camera.detach() # Get the root directory of the ros package, this will contain the models deepracer_path = rospkg.RosPack().get_path("deepracer_simulation_environment") # Grab the track data self.track_data = TrackData.get_instance() # Set all racers start position in track data self.start_positions = get_start_positions(self.racer_num) car_poses = [] for racecar_idx, racecar_name in enumerate(racecar_names): car_model_state = self.get_initial_position(racecar_name, racecar_idx) car_poses.append(car_model_state.pose) self.update_model_visual(racecar_name, self.shell_types[racecar_idx], self.car_colors[racecar_idx]) # Let KVS collect a few frames before pausing the physics, so the car # will appear on the track time.sleep(1) pause_physics = ServiceProxyWrapper('/gazebo/pause_physics', Empty) logger.info("Pausing physics after initializing the cars") pause_physics(EmptyRequest()) for racecar_name, car_pose in zip(racecar_names, car_poses): main_cameras[racecar_name].spawn_model(car_pose, os.path.join(deepracer_path, "models", "camera", "model.sdf")) logger.info("Spawning sub camera model") # Spawn the top camera model sub_camera.spawn_model(None, os.path.join(deepracer_path, "models", "top_camera", "model.sdf"))
def main(): """ Main function for tournament worker """ parser = argparse.ArgumentParser() parser.add_argument('-p', '--preset', help="(string) Name of a preset to run \ (class name from the 'presets' directory.)", type=str, required=False) parser.add_argument('--s3_bucket', help='list(string) S3 bucket', type=str, nargs='+', default=rospy.get_param("MODEL_S3_BUCKET", ["gsaur-test"])) parser.add_argument('--s3_prefix', help='list(string) S3 prefix', type=str, nargs='+', default=rospy.get_param("MODEL_S3_PREFIX", ["sagemaker"])) parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=rospy.get_param("AWS_REGION", "us-east-1")) parser.add_argument('--number_of_trials', help='(integer) Number of trials', type=int, default=int(rospy.get_param("NUMBER_OF_TRIALS", 10))) parser.add_argument( '-c', '--local_model_directory', help='(string) Path to a folder containing a checkpoint \ to restore the model from.', type=str, default='./checkpoint') parser.add_argument('--number_of_resets', help='(integer) Number of resets', type=int, default=int(rospy.get_param("NUMBER_OF_RESETS", 0))) parser.add_argument('--penalty_seconds', help='(float) penalty second', type=float, default=float(rospy.get_param("PENALTY_SECONDS", 2.0))) parser.add_argument('--job_type', help='(string) job type', type=str, default=rospy.get_param("JOB_TYPE", "EVALUATION")) parser.add_argument('--is_continuous', help='(boolean) is continous after lap completion', type=bool, default=utils.str2bool( rospy.get_param("IS_CONTINUOUS", False))) parser.add_argument('--race_type', help='(string) Race type', type=str, default=rospy.get_param("RACE_TYPE", "TIME_TRIAL")) parser.add_argument('--off_track_penalty', help='(float) off track penalty second', type=float, default=float(rospy.get_param("OFF_TRACK_PENALTY", 2.0))) parser.add_argument('--collision_penalty', help='(float) collision penalty second', type=float, default=float(rospy.get_param("COLLISION_PENALTY", 5.0))) args = parser.parse_args() arg_s3_bucket = args.s3_bucket arg_s3_prefix = args.s3_prefix logger.info("S3 bucket: %s \n S3 prefix: %s", arg_s3_bucket, arg_s3_prefix) # tournament_worker: names to be displayed in MP4. # This is racer alias in tournament worker case. display_names = utils.get_video_display_name() metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET') metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY') arg_s3_bucket, arg_s3_prefix = utils.force_list( arg_s3_bucket), utils.force_list(arg_s3_prefix) metrics_s3_buckets = utils.force_list(metrics_s3_buckets) metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys) validate_list = [ arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets, metrics_s3_object_keys ] simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None) mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None) if simtrace_s3_bucket: simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX') simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket) simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix) validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix]) if mp4_s3_bucket: mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX') mp4_s3_bucket = utils.force_list(mp4_s3_bucket) mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix) validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix]) if not all([lambda x: len(x) == len(validate_list[0]), validate_list]): log_and_exit( "Tournament worker error: Incorrect arguments passed: {}".format( validate_list), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT: raise GenericRolloutException( "number of resets is less than {}".format(MIN_RESET_COUNT)) # Instantiate Cameras if len(arg_s3_bucket) == 1: configure_camera(namespaces=['racecar']) else: configure_camera(namespaces=[ 'racecar_{}'.format(str(agent_index)) for agent_index in range(len(arg_s3_bucket)) ]) agent_list = list() s3_bucket_dict = dict() s3_prefix_dict = dict() s3_writers = list() start_positions = get_start_positions(len(arg_s3_bucket)) done_condition = utils.str_to_done_condition( rospy.get_param("DONE_CONDITION", any)) park_positions = utils.pos_2d_str_to_list( rospy.get_param("PARK_POSITIONS", [])) # if not pass in park positions for all done condition case, use default if not park_positions: park_positions = [DEFAULT_PARK_POSITION for _ in arg_s3_bucket] # tournament_worker: list of required S3 locations simtrace_s3_bucket_dict = dict() simtrace_s3_prefix_dict = dict() metrics_s3_bucket_dict = dict() metrics_s3_obect_key_dict = dict() mp4_s3_bucket_dict = dict() mp4_s3_object_prefix_dict = dict() for agent_index, s3_bucket_val in enumerate(arg_s3_bucket): agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format( str(agent_index)) racecar_name = 'racecar' if len( arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index)) s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index] s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index] # tournament_worker: remap key with agent_name instead of agent_index for list of S3 locations. simtrace_s3_bucket_dict[agent_name] = simtrace_s3_bucket[agent_index] simtrace_s3_prefix_dict[agent_name] = simtrace_s3_object_prefix[ agent_index] metrics_s3_bucket_dict[agent_name] = metrics_s3_buckets[agent_index] metrics_s3_obect_key_dict[agent_name] = metrics_s3_object_keys[ agent_index] mp4_s3_bucket_dict[agent_name] = mp4_s3_bucket[agent_index] mp4_s3_object_prefix_dict[agent_name] = mp4_s3_object_prefix[ agent_index] s3_client = SageS3Client(bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], aws_region=args.aws_region) # Load the model metadata if not os.path.exists(os.path.join(CUSTOM_FILES_PATH, agent_name)): os.makedirs(os.path.join(CUSTOM_FILES_PATH, agent_name)) model_metadata_local_path = os.path.join( os.path.join(CUSTOM_FILES_PATH, agent_name), 'model_metadata.json') utils.load_model_metadata( s3_client, os.path.normpath("%s/model/model_metadata.json" % arg_s3_prefix[agent_index]), model_metadata_local_path) # Handle backward compatibility _, _, version = parse_model_metadata(model_metadata_local_path) if float(version) < float(SIMAPP_VERSION) and \ not utils.has_current_ckpnt_name(arg_s3_bucket[agent_index], arg_s3_prefix[agent_index], args.aws_region): utils.make_compatible(arg_s3_bucket[agent_index], arg_s3_prefix[agent_index], args.aws_region, SyncFiles.TRAINER_READY.value) # Select the optimal model utils.do_model_selection(s3_bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], region=args.aws_region) # Download hyperparameters from SageMaker if not os.path.exists(agent_name): os.makedirs(agent_name) hyperparameters_file_success = False hyperparams_s3_key = os.path.normpath(arg_s3_prefix[agent_index] + "/ip/hyperparameters.json") hyperparameters_file_success = s3_client.download_file( s3_key=hyperparams_s3_key, local_path=os.path.join(agent_name, "hyperparameters.json")) sm_hyperparams_dict = {} if hyperparameters_file_success: logger.info("Received Sagemaker hyperparameters successfully!") with open(os.path.join(agent_name, "hyperparameters.json")) as file: sm_hyperparams_dict = json.load(file) else: logger.info("SageMaker hyperparameters not found.") agent_config = { 'model_metadata': model_metadata_local_path, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [ link_name.replace('racecar', racecar_name) for link_name in LINK_NAMES ], ConfigParams.VELOCITY_LIST.value: [ velocity_topic.replace('racecar', racecar_name) for velocity_topic in VELOCITY_TOPICS ], ConfigParams.STEERING_LIST.value: [ steering_topic.replace('racecar', racecar_name) for steering_topic in STEERING_TOPICS ], ConfigParams.CHANGE_START.value: utils.str2bool(rospy.get_param('CHANGE_START_POSITION', False)), ConfigParams.ALT_DIR.value: utils.str2bool( rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)), ConfigParams.ACTION_SPACE_PATH.value: 'custom_files/' + agent_name + '/model_metadata.json', ConfigParams.REWARD.value: reward_function, ConfigParams.AGENT_NAME.value: racecar_name, ConfigParams.VERSION.value: version, ConfigParams.NUMBER_OF_RESETS.value: args.number_of_resets, ConfigParams.PENALTY_SECONDS.value: args.penalty_seconds, ConfigParams.NUMBER_OF_TRIALS.value: args.number_of_trials, ConfigParams.IS_CONTINUOUS.value: args.is_continuous, ConfigParams.RACE_TYPE.value: args.race_type, ConfigParams.COLLISION_PENALTY.value: args.collision_penalty, ConfigParams.OFF_TRACK_PENALTY.value: args.off_track_penalty, ConfigParams.START_POSITION.value: start_positions[agent_index], ConfigParams.DONE_CONDITION.value: done_condition } } metrics_s3_config = { MetricsS3Keys.METRICS_BUCKET.value: metrics_s3_buckets[agent_index], MetricsS3Keys.METRICS_KEY.value: metrics_s3_object_keys[agent_index], MetricsS3Keys.ENDPOINT_URL.value: rospy.get_param('S3_ENDPOINT_URL', None), # Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed # or default argument set MetricsS3Keys.REGION.value: args.aws_region } aws_region = rospy.get_param('AWS_REGION', args.aws_region) s3_writer_job_info = [] if simtrace_s3_bucket: s3_writer_job_info.append( IterationData( 'simtrace', simtrace_s3_bucket[agent_index], simtrace_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. SIM_TRACE_EVALUATION_LOCAL_FILE.value))) if mp4_s3_bucket: s3_writer_job_info.extend([ IterationData( 'pip', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_PIP_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( '45degree', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_45DEGREE_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( 'topview', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_TOPVIEW_MP4_VALIDATION_LOCAL_PATH.value)) ]) s3_writers.append(S3Writer(job_info=s3_writer_job_info)) run_phase_subject = RunPhaseSubject() agent_list.append( create_rollout_agent( agent_config, EvalMetrics(agent_name, metrics_s3_config, args.is_continuous), run_phase_subject)) agent_list.append(create_obstacles_agent()) agent_list.append(create_bot_cars_agent()) # ROS service to indicate all the robomaker markov packages are ready for consumption signal_robomaker_markov_package_ready() PhaseObserver('/agent/training_phase', run_phase_subject) enable_domain_randomization = utils.str2bool( rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False)) graph_manager, _ = get_graph_manager( hp_dict=sm_hyperparams_dict, agent_list=agent_list, run_phase_subject=run_phase_subject, enable_domain_randomization=enable_domain_randomization, done_condition=done_condition) ds_params_instance = S3BotoDataStoreParameters( aws_region=args.aws_region, bucket_names=s3_bucket_dict, base_checkpoint_dir=args.local_model_directory, s3_folders=s3_prefix_dict) graph_manager.data_store = S3BotoDataStore(params=ds_params_instance, graph_manager=graph_manager, ignore_lock=True) graph_manager.env_params.seed = 0 task_parameters = TaskParameters() task_parameters.checkpoint_restore_path = args.local_model_directory tournament_worker(graph_manager=graph_manager, number_of_trials=args.number_of_trials, task_parameters=task_parameters, s3_writers=s3_writers, is_continuous=args.is_continuous, park_positions=park_positions) # tournament_worker: write race report to local file. write_race_report(graph_manager, model_s3_bucket_map=s3_bucket_dict, model_s3_prefix_map=s3_prefix_dict, metrics_s3_bucket_map=metrics_s3_bucket_dict, metrics_s3_key_map=metrics_s3_obect_key_dict, simtrace_s3_bucket_map=simtrace_s3_bucket_dict, simtrace_s3_prefix_map=simtrace_s3_prefix_dict, mp4_s3_bucket_map=mp4_s3_bucket_dict, mp4_s3_prefix_map=mp4_s3_object_prefix_dict, display_names=display_names) # tournament_worker: terminate tournament_race_node. terminate_tournament_race()