def _update_body_shell(self): '''update body shell type in yaml dict and then overwrite local yaml file with updated body shell type. This is for backward compatibility when body shell is not pasted in for F1 with specific racer alias or display name''' # List of body shell types body_shell_types = force_list(self._yaml_values.get(YamlKey.BODY_SHELL_TYPE_YAML_KEY.value, [None])) racer_names = force_list(self._yaml_values.get(YamlKey.RACER_NAME_YAML_KEY.value, [None])) display_names = force_list(self._yaml_values.get(YamlKey.DISPLAY_NAME_YAML_KEY.value, [None])) # If body_shell_types contains only None, figure out shell based on names # otherwise use body_shell_type defined in body_shell_types if None in body_shell_types: # use default shells only if both RACER_NAME and DISPLAY_NAME are empty if None in racer_names and None in display_names: body_shell_types = [BodyShellType.DEFAULT.value] *\ len(force_list(self._yaml_values[self._model_s3_bucket_yaml_key])) else: # If RACER_NAME is empty, use DISPLAY_NAME to get racer_alias, # and check racer_alias in F1_SHELL_USERS_LIST whether to use F1 shell or not, # otherwise use RACER_NAME as racer_alias to figure out # whether to use the f1 body shell. if None in racer_names: body_shell_types = [BodyShellType.F1_2021.value if racer_alias in F1_SHELL_USERS_LIST else BodyShellType.DEFAULT.value for racer_alias in display_names] else: body_shell_types = [BodyShellType.F1_2021.value if racer_alias in F1_SHELL_USERS_LIST else BodyShellType.DEFAULT.value for racer_alias in racer_names] self._yaml_values[YamlKey.BODY_SHELL_TYPE_YAML_KEY.value] = body_shell_types # override local yaml file with updated BODY_SHELL_TYPE self._overwrite_local_yaml_file()
def __init__(self, horizontal_fov, padding_percent, image_width, image_height, racecars_info): """ Camera information is required to map the (x, y) value of the agent on the camera image. This is because each track will have its own FOV and padding percent because to avoid Z-fighting. Once the camera position, padding percentage is available. We can map the (x,y) of the agent with respect to the track to a new plane of the camera image with padding. Arguments: horizontal_fov (float): Horizontal field of view of the camera for the given track padding_percent (float): The padding percentage to cover the whole track and look pretty image_width (int): Image width from the camera image_height (int): Image width from the camera racecars_info (list): This is the list of dicts of racecars on the track """ self.horizontal_fov = horizontal_fov self.padding_percent = padding_percent self.image_width = image_width self.image_height = image_height self.racecars_info = force_list(racecars_info) self.model_imgs = self._get_all_models_info() # Track information is required get the track bounds (xmin, xmax, ymin, ymax) # Also the agent location for that instant self.track_data = TrackData.get_instance() # This maps the (x, y) of an agent in track frame to the top camera image frame self.initialize_parameters()
def test_force_list(s3_bucket): """This function checks the functionality of force_list function in markov/utils.py Args: s3_bucket (String): S3_BUCKET """ assert utils.force_list(s3_bucket) == [s3_bucket]
def get_racecars_info(racecar_names): """ This function returns the agents information like name, car color, display name Arguments: racecar_names (str): comma seperated racecar names Returns: (list): Racecar information such as name, car color, display name """ racecars = racecar_names.split(',') racecars_info = list() racecars_color = force_list(rospy.get_param("CAR_COLOR", DEFAULT_COLOR)) racecars_display_name = force_list(rospy.get_param("DISPLAY_NAME", "")) for i, racecar_name in enumerate(racecars): racecar_dict = dict() racecar_dict['name'] = racecar_name racecar_dict['racecar_color'] = RaceCarColorToRGB[ racecars_color[i]].value racecar_dict['display_name'] = racecars_display_name[i] racecars_info.append(racecar_dict) return racecars_info
def _postprocess_yaml_values(self): '''postprocess yaml values First, it will force all FORCE_LIST_PARAMS to a list. Second, MODEL_METADATA_FILE_S3_YAML_KEY does not exist in eval, so we have to manually write it. ''' # Forcing all mandatory yaml param to list for params in self._mandatory_yaml_key: if params in self._yaml_values: self._yaml_values[params] = force_list( self._yaml_values[params]) #TODO: delete this logic if cloud service team can always pass model metadata # populate the model_metadata_s3_key values to handle both # training and evaluation for all race_formats if YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value not in self._yaml_values: # MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during # evaluation Assume model_metadata.json is present in the s3_prefix/model/ folder self._yaml_values[ YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value] = list() for s3_prefix in self._yaml_values[self._model_s3_prefix_yaml_key]: self._yaml_values[ YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value].append( os.path.join(s3_prefix, MODEL_METADATA_S3_POSTFIX)) # set model s3 buckets self._model_s3_buckets = self._yaml_values[ self._model_s3_bucket_yaml_key] # set model metadata self._model_metadata_s3_keys = \ self._yaml_values[YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value] # set body shell types self._body_shell_types = self._yaml_values[ YamlKey.BODY_SHELL_TYPE_YAML_KEY.value] # set multicar value if there is more than one value in self._model_s3_bucket_yaml_key. self._is_multicar = len( self._yaml_values[self._model_s3_bucket_yaml_key]) > 1 # set f1 as true if RACE_TYPE is F1 self._is_f1 = self._yaml_values.get( YamlKey.RACE_TYPE_YAML_KEY.value, RaceType.TIME_TRIAL.value) == F1_RACE_TYPE
def __init__(self, horizontal_fov, padding_percent, image_width, image_height, racecars_info): """ Camera information is required to map the (x, y) value of the agent on the camera image. This is because each track will have its own FOV and padding percent because to avoid Z-fighting. Once the camera position, padding percentage is available. We can map the (x,y) of the agent with respect to the track to a new plane of the camera image with padding. Arguments: horizontal_fov (float): Horizontal field of view of the camera for the given track padding_percent (float): The padding percentage to cover the whole track and look pretty image_width (int): Image width from the camera image_height (int): Image width from the camera racecars_info (list): This is the list of dicts of racecars on the track """ self.horizontal_fov = horizontal_fov self.padding_percent = padding_percent self.image_width = image_width self.image_height = image_height self.racecars_info = force_list(racecars_info) rospy.wait_for_service('/gazebo/get_model_state') # We have no guarantees as to when gazebo will load the model, therefore we need # to wait until the model is loaded prior to resetting it for the first time self.get_model_client = ServiceProxyWrapper('/gazebo/get_model_state', GetModelState) self.model_names, self.model_imgs = self._get_all_models_info() wait_for_model = True while wait_for_model: # If the model is not loaded the get model service will log an error # Therefore, we add an timer to prevent the log from getting spammed with # errors time.sleep(WAIT_TO_PREVENT_SPAM) model_status = list() for model_name in self.model_names: model = self.get_model_client(model_name, '') model_status.append(model.success) wait_for_model = not all(model_status) # Track information is required get the track bounds (xmin, xmax, ymin, ymax) # Also the agent location for that instant self.track_data = TrackData.get_instance() # This maps the (x, y) of an agent in track frame to the top camera image frame self.initialize_parameters()
def get_racecars_info(racecar_names): """ This function returns the agents information like name, car color, display name Arguments: racecar_names (list): comma seperated racecar names Returns: (list): Racecar information such as name, car color, display name """ racecars = racecar_names racecars_info = list() racecars_color = force_list( rospy.get_param("CAR_COLOR", [DEFAULT_COLOR] * len(racecar_names))) racecars_display_name = get_video_display_name() for i, racecar_name in enumerate(racecars): racecar_dict = dict() racecar_dict['name'] = racecar_name racecar_dict['racecar_color'] = racecars_color[i] racecar_dict['display_name'] = racecars_display_name[i] racecars_info.append(racecar_dict) return racecars_info
def __init__(self, horizontal_fov, padding_percent, image_width, image_height, racecars_info): """ Camera information is required to map the (x, y) value of the agent on the camera image. This is because each track will have its own FOV and padding percent because to avoid Z-fighting. Once the camera position, padding percentage is available. We can map the (x,y) of the agent with respect to the track to a new plane of the camera image with padding. Arguments: horizontal_fov (float): Horizontal field of view of the camera for the given track padding_percent (float): The padding percentage to cover the whole track and look pretty image_width (int): Image width from the camera image_height (int): Image width from the camera racecars_info (list): This is the list of dicts of racecars on the track """ self.horizontal_fov = horizontal_fov self.padding_percent = padding_percent self.image_width = image_width self.image_height = image_height self.racecars_info = force_list(racecars_info) # # We have no guarantees as to when gazebo will load the model, therefore we need # to wait until the model is loaded and markov packages has spawned all the models # rospy.wait_for_service('/gazebo/get_model_state') rospy.wait_for_service('/robomaker_markov_package_ready') self.get_model_client = ServiceProxyWrapper('/gazebo/get_model_state', GetModelState) self.model_names, self.model_imgs = self._get_all_models_info() # Track information is required get the track bounds (xmin, xmax, ymin, ymax) # Also the agent location for that instant self.track_data = TrackData.get_instance() # This maps the (x, y) of an agent in track frame to the top camera image frame self.initialize_parameters()
def main(): """ Main function for tournament worker """ parser = argparse.ArgumentParser() parser.add_argument('-p', '--preset', help="(string) Name of a preset to run \ (class name from the 'presets' directory.)", type=str, required=False) parser.add_argument('--s3_bucket', help='list(string) S3 bucket', type=str, nargs='+', default=rospy.get_param("MODEL_S3_BUCKET", ["gsaur-test"])) parser.add_argument('--s3_prefix', help='list(string) S3 prefix', type=str, nargs='+', default=rospy.get_param("MODEL_S3_PREFIX", ["sagemaker"])) parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=rospy.get_param("AWS_REGION", "us-east-1")) parser.add_argument('--number_of_trials', help='(integer) Number of trials', type=int, default=int(rospy.get_param("NUMBER_OF_TRIALS", 10))) parser.add_argument( '-c', '--local_model_directory', help='(string) Path to a folder containing a checkpoint \ to restore the model from.', type=str, default='./checkpoint') parser.add_argument('--number_of_resets', help='(integer) Number of resets', type=int, default=int(rospy.get_param("NUMBER_OF_RESETS", 0))) parser.add_argument('--penalty_seconds', help='(float) penalty second', type=float, default=float(rospy.get_param("PENALTY_SECONDS", 2.0))) parser.add_argument('--job_type', help='(string) job type', type=str, default=rospy.get_param("JOB_TYPE", "EVALUATION")) parser.add_argument('--is_continuous', help='(boolean) is continous after lap completion', type=bool, default=utils.str2bool( rospy.get_param("IS_CONTINUOUS", False))) parser.add_argument('--race_type', help='(string) Race type', type=str, default=rospy.get_param("RACE_TYPE", "TIME_TRIAL")) parser.add_argument('--off_track_penalty', help='(float) off track penalty second', type=float, default=float(rospy.get_param("OFF_TRACK_PENALTY", 2.0))) parser.add_argument('--collision_penalty', help='(float) collision penalty second', type=float, default=float(rospy.get_param("COLLISION_PENALTY", 5.0))) args = parser.parse_args() arg_s3_bucket = args.s3_bucket arg_s3_prefix = args.s3_prefix logger.info("S3 bucket: %s \n S3 prefix: %s", arg_s3_bucket, arg_s3_prefix) # tournament_worker: names to be displayed in MP4. # This is racer alias in tournament worker case. display_names = rospy.get_param('DISPLAY_NAME', "") metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET') metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY') arg_s3_bucket, arg_s3_prefix = utils.force_list( arg_s3_bucket), utils.force_list(arg_s3_prefix) metrics_s3_buckets = utils.force_list(metrics_s3_buckets) metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys) validate_list = [ arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets, metrics_s3_object_keys ] simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None) mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None) if simtrace_s3_bucket: simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX') simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket) simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix) validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix]) if mp4_s3_bucket: mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX') mp4_s3_bucket = utils.force_list(mp4_s3_bucket) mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix) validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix]) if not all([lambda x: len(x) == len(validate_list[0]), validate_list]): utils.log_and_exit( "Eval worker error: Incorrect arguments passed: {}".format( validate_list), utils.SIMAPP_SIMULATION_WORKER_EXCEPTION, utils.SIMAPP_EVENT_ERROR_CODE_500) if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT: raise GenericRolloutException( "number of resets is less than {}".format(MIN_RESET_COUNT)) # Instantiate Cameras if len(arg_s3_bucket) == 1: configure_camera(namespaces=['racecar']) else: configure_camera(namespaces=[ 'racecar_{}'.format(str(agent_index)) for agent_index in range(len(arg_s3_bucket)) ]) agent_list = list() s3_bucket_dict = dict() s3_prefix_dict = dict() s3_writers = list() # tournament_worker: list of required S3 locations simtrace_s3_bucket_dict = dict() simtrace_s3_prefix_dict = dict() metrics_s3_bucket_dict = dict() metrics_s3_obect_key_dict = dict() mp4_s3_bucket_dict = dict() mp4_s3_object_prefix_dict = dict() for agent_index, s3_bucket_val in enumerate(arg_s3_bucket): agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format( str(agent_index)) racecar_name = 'racecar' if len( arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index)) s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index] s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index] # tournament_worker: remap key with agent_name instead of agent_index for list of S3 locations. simtrace_s3_bucket_dict[agent_name] = simtrace_s3_bucket[agent_index] simtrace_s3_prefix_dict[agent_name] = simtrace_s3_object_prefix[ agent_index] metrics_s3_bucket_dict[agent_name] = metrics_s3_buckets[agent_index] metrics_s3_obect_key_dict[agent_name] = metrics_s3_object_keys[ agent_index] mp4_s3_bucket_dict[agent_name] = mp4_s3_bucket[agent_index] mp4_s3_object_prefix_dict[agent_name] = mp4_s3_object_prefix[ agent_index] s3_client = SageS3Client(bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], aws_region=args.aws_region) # Load the model metadata if not os.path.exists(os.path.join(CUSTOM_FILES_PATH, agent_name)): os.makedirs(os.path.join(CUSTOM_FILES_PATH, agent_name)) model_metadata_local_path = os.path.join( os.path.join(CUSTOM_FILES_PATH, agent_name), 'model_metadata.json') utils.load_model_metadata( s3_client, os.path.normpath("%s/model/model_metadata.json" % arg_s3_prefix[agent_index]), model_metadata_local_path) # Handle backward compatibility _, _, version = parse_model_metadata(model_metadata_local_path) if float(version) < float(utils.SIMAPP_VERSION) and \ not utils.has_current_ckpnt_name(arg_s3_bucket[agent_index], arg_s3_prefix[agent_index], args.aws_region): utils.make_compatible(arg_s3_bucket[agent_index], arg_s3_prefix[agent_index], args.aws_region, SyncFiles.TRAINER_READY.value) # Select the optimal model utils.do_model_selection(s3_bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], region=args.aws_region) # Download hyperparameters from SageMaker if not os.path.exists(agent_name): os.makedirs(agent_name) hyperparameters_file_success = False hyperparams_s3_key = os.path.normpath(arg_s3_prefix[agent_index] + "/ip/hyperparameters.json") hyperparameters_file_success = s3_client.download_file( s3_key=hyperparams_s3_key, local_path=os.path.join(agent_name, "hyperparameters.json")) sm_hyperparams_dict = {} if hyperparameters_file_success: logger.info("Received Sagemaker hyperparameters successfully!") with open(os.path.join(agent_name, "hyperparameters.json")) as file: sm_hyperparams_dict = json.load(file) else: logger.info("SageMaker hyperparameters not found.") agent_config = { 'model_metadata': model_metadata_local_path, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [ link_name.replace('racecar', racecar_name) for link_name in LINK_NAMES ], ConfigParams.VELOCITY_LIST.value: [ velocity_topic.replace('racecar', racecar_name) for velocity_topic in VELOCITY_TOPICS ], ConfigParams.STEERING_LIST.value: [ steering_topic.replace('racecar', racecar_name) for steering_topic in STEERING_TOPICS ], ConfigParams.CHANGE_START.value: utils.str2bool(rospy.get_param('CHANGE_START_POSITION', False)), ConfigParams.ALT_DIR.value: utils.str2bool( rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)), ConfigParams.ACTION_SPACE_PATH.value: 'custom_files/' + agent_name + '/model_metadata.json', ConfigParams.REWARD.value: reward_function, ConfigParams.AGENT_NAME.value: racecar_name, ConfigParams.VERSION.value: version, ConfigParams.NUMBER_OF_RESETS.value: args.number_of_resets, ConfigParams.PENALTY_SECONDS.value: args.penalty_seconds, ConfigParams.NUMBER_OF_TRIALS.value: args.number_of_trials, ConfigParams.IS_CONTINUOUS.value: args.is_continuous, ConfigParams.RACE_TYPE.value: args.race_type, ConfigParams.COLLISION_PENALTY.value: args.collision_penalty, ConfigParams.OFF_TRACK_PENALTY.value: args.off_track_penalty } } metrics_s3_config = { MetricsS3Keys.METRICS_BUCKET.value: metrics_s3_buckets[agent_index], MetricsS3Keys.METRICS_KEY.value: metrics_s3_object_keys[agent_index], # Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed # or default argument set MetricsS3Keys.REGION.value: args.aws_region, # Replaced rospy.get_param('MODEL_S3_BUCKET') to be equal to the argument being passed # or default argument set MetricsS3Keys.STEP_BUCKET.value: arg_s3_bucket[agent_index], # Replaced rospy.get_param('MODEL_S3_PREFIX') to be equal to the argument being passed # or default argument set MetricsS3Keys.STEP_KEY.value: os.path.join(arg_s3_prefix[agent_index], EVALUATION_SIMTRACE_DATA_S3_OBJECT_KEY) } aws_region = rospy.get_param('AWS_REGION', args.aws_region) s3_writer_job_info = [] if simtrace_s3_bucket: s3_writer_job_info.append( IterationData( 'simtrace', simtrace_s3_bucket[agent_index], simtrace_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. SIM_TRACE_EVALUATION_LOCAL_FILE.value))) if mp4_s3_bucket: s3_writer_job_info.extend([ IterationData( 'pip', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_PIP_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( '45degree', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_45DEGREE_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( 'topview', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_TOPVIEW_MP4_VALIDATION_LOCAL_PATH.value)) ]) s3_writers.append(S3Writer(job_info=s3_writer_job_info)) run_phase_subject = RunPhaseSubject() agent_list.append( create_rollout_agent(agent_config, EvalMetrics(agent_name, metrics_s3_config), run_phase_subject)) agent_list.append(create_obstacles_agent()) agent_list.append(create_bot_cars_agent()) # ROS service to indicate all the robomaker markov packages are ready for consumption signal_robomaker_markov_package_ready() PhaseObserver('/agent/training_phase', run_phase_subject) graph_manager, _ = get_graph_manager(hp_dict=sm_hyperparams_dict, agent_list=agent_list, run_phase_subject=run_phase_subject) ds_params_instance = S3BotoDataStoreParameters( aws_region=args.aws_region, bucket_names=s3_bucket_dict, base_checkpoint_dir=args.local_model_directory, s3_folders=s3_prefix_dict) graph_manager.data_store = S3BotoDataStore(params=ds_params_instance, graph_manager=graph_manager, ignore_lock=True) graph_manager.env_params.seed = 0 task_parameters = TaskParameters() task_parameters.checkpoint_restore_path = args.local_model_directory tournament_worker(graph_manager=graph_manager, number_of_trials=args.number_of_trials, task_parameters=task_parameters, s3_writers=s3_writers, is_continuous=args.is_continuous) # tournament_worker: write race report to local file. write_race_report(graph_manager, model_s3_bucket_map=s3_bucket_dict, model_s3_prefix_map=s3_prefix_dict, metrics_s3_bucket_map=metrics_s3_bucket_dict, metrics_s3_key_map=metrics_s3_obect_key_dict, simtrace_s3_bucket_map=simtrace_s3_bucket_dict, simtrace_s3_prefix_map=simtrace_s3_prefix_dict, mp4_s3_bucket_map=mp4_s3_bucket_dict, mp4_s3_prefix_map=mp4_s3_object_prefix_dict, display_names=display_names) # tournament_worker: terminate tournament_race_node. terminate_tournament_race()
def main(): """ Main function for tournament""" try: # parse argument s3_region = sys.argv[1] s3_bucket = sys.argv[2] s3_prefix = sys.argv[3] s3_yaml_name = sys.argv[4] # create boto3 session/client and download yaml/json file session = boto3.session.Session() s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None) s3_client = session.client('s3', region_name=s3_region, endpoint_url=s3_endpoint_url, config=get_boto_config()) yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name)) local_yaml_path = os.path.abspath( os.path.join(os.getcwd(), s3_yaml_name)) s3_client.download_file(Bucket=s3_bucket, Key=yaml_key, Filename=local_yaml_path) # Intermediate tournament files queue_pickle_name = 'tournament_candidate_queue.pkl' queue_pickle_s3_key = os.path.normpath( os.path.join(s3_prefix, queue_pickle_name)) local_queue_pickle_path = os.path.abspath( os.path.join(os.getcwd(), queue_pickle_name)) report_pickle_name = 'tournament_report.pkl' report_pickle_s3_key = os.path.normpath( os.path.join(s3_prefix, report_pickle_name)) local_report_pickle_path = os.path.abspath( os.path.join(os.getcwd(), report_pickle_name)) final_report_name = 'tournament_report.json' final_report_s3_key = os.path.normpath( os.path.join(s3_prefix, final_report_name)) try: s3_client.download_file(Bucket=s3_bucket, Key=queue_pickle_s3_key, Filename=local_queue_pickle_path) s3_client.download_file(Bucket=s3_bucket, Key=report_pickle_s3_key, Filename=local_report_pickle_path) except: pass # Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing yaml_dict = get_yaml_dict(local_yaml_path) # Forcing the yaml parameter to list force_list_params = [ MODEL_S3_BUCKET_YAML_KEY, MODEL_S3_PREFIX_YAML_KEY, MODEL_METADATA_FILE_S3_YAML_KEY, METRICS_S3_BUCKET_YAML_KEY, METRICS_S3_PREFIX_YAML_KEY, SIMTRACE_S3_BUCKET_YAML_KEY, SIMTRACE_S3_PREFIX_YAML_KEY, MP4_S3_BUCKET_YAML_KEY, MP4_S3_PREFIX_YAML_KEY, DISPLAY_NAME_YAML_KEY ] for params in force_list_params: yaml_dict[params] = force_list(yaml_dict.get(params, None)) # Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats if None in yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY]: # MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation # Assume model_metadata.json is present in the s3_prefix/model/ folder yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY] = list() for s3_prefix in yaml_dict[MODEL_S3_PREFIX_YAML_KEY]: yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY].append( os.path.join(s3_prefix, 'model/model_metadata.json')) # Validate the yaml values validate_yaml_values(yaml_dict) if os.path.exists(local_queue_pickle_path): with open(local_queue_pickle_path, 'rb') as f: tournament_candidate_queue = pickle.load(f) with open(local_report_pickle_path, 'rb') as f: tournament_report = pickle.load(f) logger.info('tournament_candidate_queue loaded from existing file') else: logger.info('tournament_candidate_queue initialized') tournament_candidate_queue = deque() for agent_idx, _ in enumerate(yaml_dict[MODEL_S3_BUCKET_YAML_KEY]): tournament_candidate_queue.append( (yaml_dict[MODEL_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[MODEL_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY][agent_idx], yaml_dict[METRICS_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[METRICS_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[SIMTRACE_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[SIMTRACE_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[MP4_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[MP4_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[DISPLAY_NAME_YAML_KEY][agent_idx])) tournament_report = [] race_idx = len(tournament_report) while len(tournament_candidate_queue) > 1: car1 = tournament_candidate_queue.popleft() car2 = tournament_candidate_queue.popleft() (car1_model_s3_bucket, car1_s3_prefix, car1_model_metadata, car1_metrics_bucket, car1_metrics_s3_key, car1_simtrace_bucket, car1_simtrace_prefix, car1_mp4_bucket, car1_mp4_prefix, car1_display_name) = car1 (car2_model_s3_bucket, car2_s3_prefix, car2_model_metadata, car2_metrics_bucket, car2_metrics_s3_key, car2_simtrace_bucket, car2_simtrace_prefix, car2_mp4_bucket, car2_mp4_prefix, car2_display_name) = car2 race_yaml_dict = generate_race_yaml(yaml_dict=yaml_dict, car1=car1, car2=car2, race_idx=race_idx) race_car_colors = ["Orange", "Purple"] race_model_s3_buckets = [ car1_model_s3_bucket, car2_model_s3_bucket ] race_model_metadatas = [car1_model_metadata, car2_model_metadata] # List of directories created dirs_to_delete = list() yaml_dir = os.path.abspath(os.path.join(os.getcwd(), str(race_idx))) os.makedirs(yaml_dir) dirs_to_delete.append(yaml_dir) race_yaml_path = os.path.abspath( os.path.join(yaml_dir, 'evaluation_params.yaml')) with open(race_yaml_path, 'w') as race_yaml_file: yaml.dump(race_yaml_dict, race_yaml_file) # List of racecar names that should include second camera while launching racecars_with_stereo_cameras = list() # List of racecar names that should include lidar while launching racecars_with_lidars = list() # List of SimApp versions simapp_versions = list() for agent_index, model_s3_bucket in enumerate( race_model_s3_buckets): racecar_name = 'racecar_' + str(agent_index) # Make a local folder with the racecar name to download the model_metadata.json os.makedirs(os.path.join(os.getcwd(), racecar_name)) dirs_to_delete.append(os.path.join(os.getcwd(), racecar_name)) local_model_metadata_path = os.path.abspath( os.path.join(os.path.join(os.getcwd(), racecar_name), 'model_metadata.json')) json_key = race_model_metadatas[agent_index] json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '') s3_client.download_file(Bucket=model_s3_bucket, Key=json_key, Filename=local_model_metadata_path) sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata( local_model_metadata_path) simapp_versions.append(simapp_version) if Input.STEREO.value in sensors: racecars_with_stereo_cameras.append(racecar_name) if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors: racecars_with_lidars.append(racecar_name) cmd = [ os.path.join(os.path.dirname(os.path.abspath(__file__)), "tournament_race_node.py"), str(race_idx), race_yaml_path, ','.join(racecars_with_stereo_cameras), ','.join(racecars_with_lidars), ','.join(race_car_colors), ','.join(simapp_versions) ] try: return_code, _, stderr = run_cmd(cmd_args=cmd, shell=False, stdout=None, stderr=None) except KeyboardInterrupt: logger.info( "KeyboardInterrupt raised, SimApp must be faulted! exiting..." ) return # Retrieve winner and append tournament report with open('race_report.pkl', 'rb') as f: race_report = pickle.load(f) race_report['race_idx'] = race_idx winner = car1 if race_report[ 'winner'] == car1_display_name else car2 logger.info("race {}'s winner: {}".format(race_idx, race_report['winner'])) tournament_candidate_queue.append(winner) tournament_report.append(race_report) # Clean up directories created for dir_to_delete in dirs_to_delete: shutil.rmtree(dir_to_delete, ignore_errors=True) race_idx += 1 # Persist latest queue and report to use after job restarts. with open(local_queue_pickle_path, 'wb') as f: pickle.dump(tournament_candidate_queue, f, protocol=2) s3_client.upload_file(Filename=local_queue_pickle_path, Bucket=s3_bucket, Key=queue_pickle_s3_key) with open(local_report_pickle_path, 'wb') as f: pickle.dump(tournament_report, f, protocol=2) s3_client.upload_file(Filename=local_report_pickle_path, Bucket=s3_bucket, Key=report_pickle_s3_key) # If there is more than 1 candidates then restart the simulation job otherwise # tournament is finished, persists final report and ends the job. if len(tournament_candidate_queue) > 1: restart_simulation_job( os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'), s3_region) break else: # Persist final tournament report in json format # and terminate the job by canceling it s3_client.put_object(Bucket=s3_bucket, Key=final_report_s3_key, Body=json.dumps(tournament_report)) cancel_simulation_job( os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'), s3_region) except Exception as e: log_and_exit( "Tournament node failed: s3_bucket: {}, yaml_key: {}, {}".format( s3_bucket, yaml_key, e), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def main(): """ Main function for evaluation worker """ parser = argparse.ArgumentParser() parser.add_argument('-p', '--preset', help="(string) Name of a preset to run \ (class name from the 'presets' directory.)", type=str, required=False) parser.add_argument('--s3_bucket', help='list(string) S3 bucket', type=str, nargs='+', default=rospy.get_param("MODEL_S3_BUCKET", ["gsaur-test"])) parser.add_argument('--s3_prefix', help='list(string) S3 prefix', type=str, nargs='+', default=rospy.get_param("MODEL_S3_PREFIX", ["sagemaker"])) parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=rospy.get_param("AWS_REGION", "us-east-1")) parser.add_argument('--number_of_trials', help='(integer) Number of trials', type=int, default=int(rospy.get_param("NUMBER_OF_TRIALS", 10))) parser.add_argument( '-c', '--local_model_directory', help='(string) Path to a folder containing a checkpoint \ to restore the model from.', type=str, default='./checkpoint') parser.add_argument('--number_of_resets', help='(integer) Number of resets', type=int, default=int(rospy.get_param("NUMBER_OF_RESETS", 0))) parser.add_argument('--penalty_seconds', help='(float) penalty second', type=float, default=float(rospy.get_param("PENALTY_SECONDS", 2.0))) parser.add_argument('--job_type', help='(string) job type', type=str, default=rospy.get_param("JOB_TYPE", "EVALUATION")) parser.add_argument('--is_continuous', help='(boolean) is continous after lap completion', type=bool, default=utils.str2bool( rospy.get_param("IS_CONTINUOUS", False))) parser.add_argument('--race_type', help='(string) Race type', type=str, default=rospy.get_param("RACE_TYPE", "TIME_TRIAL")) parser.add_argument('--off_track_penalty', help='(float) off track penalty second', type=float, default=float(rospy.get_param("OFF_TRACK_PENALTY", 2.0))) parser.add_argument('--collision_penalty', help='(float) collision penalty second', type=float, default=float(rospy.get_param("COLLISION_PENALTY", 5.0))) args = parser.parse_args() arg_s3_bucket = args.s3_bucket arg_s3_prefix = args.s3_prefix logger.info("S3 bucket: %s \n S3 prefix: %s", arg_s3_bucket, arg_s3_prefix) metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET') metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY') arg_s3_bucket, arg_s3_prefix = utils.force_list( arg_s3_bucket), utils.force_list(arg_s3_prefix) metrics_s3_buckets = utils.force_list(metrics_s3_buckets) metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys) validate_list = [ arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets, metrics_s3_object_keys ] simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None) mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None) if simtrace_s3_bucket: simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX') simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket) simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix) validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix]) if mp4_s3_bucket: mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX') mp4_s3_bucket = utils.force_list(mp4_s3_bucket) mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix) validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix]) if not all([lambda x: len(x) == len(validate_list[0]), validate_list]): log_and_exit( "Eval worker error: Incorrect arguments passed: {}".format( validate_list), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT: raise GenericRolloutException( "number of resets is less than {}".format(MIN_RESET_COUNT)) # Instantiate Cameras if len(arg_s3_bucket) == 1: configure_camera(namespaces=['racecar']) else: configure_camera(namespaces=[ 'racecar_{}'.format(str(agent_index)) for agent_index in range(len(arg_s3_bucket)) ]) agent_list = list() s3_bucket_dict = dict() s3_prefix_dict = dict() checkpoint_dict = dict() simtrace_video_s3_writers = [] start_positions = get_start_positions(len(arg_s3_bucket)) done_condition = utils.str_to_done_condition( rospy.get_param("DONE_CONDITION", any)) park_positions = utils.pos_2d_str_to_list( rospy.get_param("PARK_POSITIONS", [])) # if not pass in park positions for all done condition case, use default if not park_positions: park_positions = [DEFAULT_PARK_POSITION for _ in arg_s3_bucket] for agent_index, _ in enumerate(arg_s3_bucket): agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format( str(agent_index)) racecar_name = 'racecar' if len( arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index)) s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index] s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index] # download model metadata model_metadata = ModelMetadata( bucket=arg_s3_bucket[agent_index], s3_key=get_s3_key(arg_s3_prefix[agent_index], MODEL_METADATA_S3_POSTFIX), region_name=args.aws_region, local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format(agent_name)) model_metadata_info = model_metadata.get_model_metadata_info() version = model_metadata_info[ModelMetadataKeys.VERSION.value] # checkpoint s3 instance checkpoint = Checkpoint(bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], region_name=args.aws_region, agent_name=agent_name, checkpoint_dir=args.local_model_directory) # make coach checkpoint compatible if version < SIMAPP_VERSION_2 and not checkpoint.rl_coach_checkpoint.is_compatible( ): checkpoint.rl_coach_checkpoint.make_compatible( checkpoint.syncfile_ready) # get best model checkpoint string model_checkpoint_name = checkpoint.deepracer_checkpoint_json.get_deepracer_best_checkpoint( ) # Select the best checkpoint model by uploading rl coach .coach_checkpoint file checkpoint.rl_coach_checkpoint.update( model_checkpoint_name=model_checkpoint_name, s3_kms_extra_args=utils.get_s3_kms_extra_args()) checkpoint_dict[agent_name] = checkpoint agent_config = { 'model_metadata': model_metadata, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [ link_name.replace('racecar', racecar_name) for link_name in LINK_NAMES ], ConfigParams.VELOCITY_LIST.value: [ velocity_topic.replace('racecar', racecar_name) for velocity_topic in VELOCITY_TOPICS ], ConfigParams.STEERING_LIST.value: [ steering_topic.replace('racecar', racecar_name) for steering_topic in STEERING_TOPICS ], ConfigParams.CHANGE_START.value: utils.str2bool(rospy.get_param('CHANGE_START_POSITION', False)), ConfigParams.ALT_DIR.value: utils.str2bool( rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)), ConfigParams.MODEL_METADATA.value: model_metadata, ConfigParams.REWARD.value: reward_function, ConfigParams.AGENT_NAME.value: racecar_name, ConfigParams.VERSION.value: version, ConfigParams.NUMBER_OF_RESETS.value: args.number_of_resets, ConfigParams.PENALTY_SECONDS.value: args.penalty_seconds, ConfigParams.NUMBER_OF_TRIALS.value: args.number_of_trials, ConfigParams.IS_CONTINUOUS.value: args.is_continuous, ConfigParams.RACE_TYPE.value: args.race_type, ConfigParams.COLLISION_PENALTY.value: args.collision_penalty, ConfigParams.OFF_TRACK_PENALTY.value: args.off_track_penalty, ConfigParams.START_POSITION.value: start_positions[agent_index], ConfigParams.DONE_CONDITION.value: done_condition } } metrics_s3_config = { MetricsS3Keys.METRICS_BUCKET.value: metrics_s3_buckets[agent_index], MetricsS3Keys.METRICS_KEY.value: metrics_s3_object_keys[agent_index], # Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed # or default argument set MetricsS3Keys.REGION.value: args.aws_region } aws_region = rospy.get_param('AWS_REGION', args.aws_region) if simtrace_s3_bucket: simtrace_video_s3_writers.append( SimtraceVideo( upload_type=SimtraceVideoNames.SIMTRACE_EVAL.value, bucket=simtrace_s3_bucket[agent_index], s3_prefix=simtrace_s3_object_prefix[agent_index], region_name=aws_region, local_path=SIMTRACE_EVAL_LOCAL_PATH_FORMAT.format( agent_name))) if mp4_s3_bucket: simtrace_video_s3_writers.extend([ SimtraceVideo( upload_type=SimtraceVideoNames.PIP.value, bucket=mp4_s3_bucket[agent_index], s3_prefix=mp4_s3_object_prefix[agent_index], region_name=aws_region, local_path=CAMERA_PIP_MP4_LOCAL_PATH_FORMAT.format( agent_name)), SimtraceVideo( upload_type=SimtraceVideoNames.DEGREE45.value, bucket=mp4_s3_bucket[agent_index], s3_prefix=mp4_s3_object_prefix[agent_index], region_name=aws_region, local_path=CAMERA_45DEGREE_LOCAL_PATH_FORMAT.format( agent_name)), SimtraceVideo( upload_type=SimtraceVideoNames.TOPVIEW.value, bucket=mp4_s3_bucket[agent_index], s3_prefix=mp4_s3_object_prefix[agent_index], region_name=aws_region, local_path=CAMERA_TOPVIEW_LOCAL_PATH_FORMAT.format( agent_name)) ]) run_phase_subject = RunPhaseSubject() agent_list.append( create_rollout_agent( agent_config, EvalMetrics(agent_name, metrics_s3_config, args.is_continuous), run_phase_subject)) agent_list.append(create_obstacles_agent()) agent_list.append(create_bot_cars_agent()) # ROS service to indicate all the robomaker markov packages are ready for consumption signal_robomaker_markov_package_ready() PhaseObserver('/agent/training_phase', run_phase_subject) enable_domain_randomization = utils.str2bool( rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False)) sm_hyperparams_dict = {} # Make the clients that will allow us to pause and unpause the physics rospy.wait_for_service('/gazebo/pause_physics_dr') rospy.wait_for_service('/gazebo/unpause_physics_dr') pause_physics = ServiceProxyWrapper('/gazebo/pause_physics_dr', Empty) unpause_physics = ServiceProxyWrapper('/gazebo/unpause_physics_dr', Empty) graph_manager, _ = get_graph_manager( hp_dict=sm_hyperparams_dict, agent_list=agent_list, run_phase_subject=run_phase_subject, enable_domain_randomization=enable_domain_randomization, done_condition=done_condition, pause_physics=pause_physics, unpause_physics=unpause_physics) ds_params_instance = S3BotoDataStoreParameters( checkpoint_dict=checkpoint_dict) graph_manager.data_store = S3BotoDataStore(params=ds_params_instance, graph_manager=graph_manager, ignore_lock=True) graph_manager.env_params.seed = 0 task_parameters = TaskParameters() task_parameters.checkpoint_restore_path = args.local_model_directory evaluation_worker(graph_manager=graph_manager, number_of_trials=args.number_of_trials, task_parameters=task_parameters, simtrace_video_s3_writers=simtrace_video_s3_writers, is_continuous=args.is_continuous, park_positions=park_positions, race_type=args.race_type, pause_physics=pause_physics, unpause_physics=unpause_physics)
def main(): """ Main function for downloading yaml params """ try: # parse argument s3_region = sys.argv[1] s3_bucket = sys.argv[2] s3_prefix = sys.argv[3] s3_yaml_name = sys.argv[4] launch_name = sys.argv[5] # create boto3 session/client and download yaml/json file session = boto3.session.Session() s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None) if s3_endpoint_url is not None: LOG.info('Endpoint URL {}'.format(s3_endpoint_url)) rospy.set_param('S3_ENDPOINT_URL', s3_endpoint_url) else: # create boto3 session/client and download yaml/json file ec2_client = session.client('ec2', s3_region) LOG.info('Checking internet connection...') response = ec2_client.describe_vpcs() if not response['Vpcs']: log_and_exit("No VPC attached to instance", SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) LOG.info('Verified internet connection') s3_client = session.client('s3', region_name=s3_region, endpoint_url=s3_endpoint_url, config=get_boto_config()) yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name)) local_yaml_path = os.path.abspath( os.path.join(os.getcwd(), s3_yaml_name)) s3_client.download_file(Bucket=s3_bucket, Key=yaml_key, Filename=local_yaml_path) # Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing default_yaml_values = { RACE_TYPE_YAML_KEY: TIME_TRIAL_RACE_TYPE, MODEL_S3_BUCKET_YAML_KEY: s3_bucket, MODEL_S3_PREFIX_YAML_KEY: s3_prefix, CAR_COLOR_YAML_KEY: DEFAULT_COLOR, MODEL_METADATA_FILE_S3_YAML_KEY: None } yaml_dict = get_yaml_dict(local_yaml_path) yaml_values = get_yaml_values(yaml_dict, default_yaml_values) # Forcing the yaml parameter to list force_list_params = [ MODEL_METADATA_FILE_S3_YAML_KEY, MODEL_S3_BUCKET_YAML_KEY, MODEL_S3_PREFIX_YAML_KEY, CAR_COLOR_YAML_KEY ] for params in force_list_params: yaml_values[params] = force_list(yaml_values[params]) # Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats if None in yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY]: # MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation # Assume model_metadata.json is present in the s3_prefix/model/ folder yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY] = list() for s3_prefix in yaml_values[MODEL_S3_PREFIX_YAML_KEY]: yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY].append( os.path.join(s3_prefix, 'model/model_metadata.json')) # Set multicar value if its a head to model racetype multicar = yaml_values[RACE_TYPE_YAML_KEY] == HEAD_TO_MODEL_RACE_TYPE # Validate the yaml values validate_yaml_values(yaml_values, multicar) # List of racecar names that should include second camera while launching racecars_with_stereo_cameras = list() # List of racecar names that should include lidar while launching racecars_with_lidars = list() # List of SimApp versions simapp_versions = list() for agent_index, model_s3_bucket in enumerate( yaml_values[MODEL_S3_BUCKET_YAML_KEY]): racecar_name = 'racecar_' + str(agent_index) if len( yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 else 'racecar' # Make a local folder with the racecar name to download the model_metadata.json if not os.path.exists(os.path.join(os.getcwd(), racecar_name)): os.makedirs(os.path.join(os.getcwd(), racecar_name)) local_model_metadata_path = os.path.abspath( os.path.join(os.path.join(os.getcwd(), racecar_name), 'model_metadata.json')) json_key = yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY][ agent_index] json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '') s3_client.download_file(Bucket=model_s3_bucket, Key=json_key, Filename=local_model_metadata_path) sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata( local_model_metadata_path) simapp_versions.append(simapp_version) if Input.STEREO.value in sensors: racecars_with_stereo_cameras.append(racecar_name) if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors: racecars_with_lidars.append(racecar_name) cmd = [ ''.join(("roslaunch deepracer_simulation_environment {} ".format( launch_name), "local_yaml_path:={} ".format(local_yaml_path), "racecars_with_stereo_cameras:={} ".format( ','.join(racecars_with_stereo_cameras)), "racecars_with_lidars:={} multicar:={} ".format( ','.join(racecars_with_lidars), multicar), "car_colors:={} simapp_versions:={}".format( ','.join(yaml_values[CAR_COLOR_YAML_KEY]), ','.join(simapp_versions)))) ] Popen(cmd, shell=True, executable="/bin/bash") except botocore.exceptions.ClientError as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400) except botocore.exceptions.EndpointConnectionError: log_and_exit("No Internet connection or s3 service unavailable", SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) except Exception as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def main(): """ Main function for evaluation worker """ parser = argparse.ArgumentParser() parser.add_argument('-p', '--preset', help="(string) Name of a preset to run \ (class name from the 'presets' directory.)", type=str, required=False) parser.add_argument('--s3_bucket', help='list(string) S3 bucket', type=str, nargs='+', default=rospy.get_param("MODEL_S3_BUCKET", ["gsaur-test"])) parser.add_argument('--s3_prefix', help='list(string) S3 prefix', type=str, nargs='+', default=rospy.get_param("MODEL_S3_PREFIX", ["sagemaker"])) parser.add_argument('--s3_endpoint_url', help='(string) S3 endpoint URL', type=str, default=rospy.get_param("S3_ENDPOINT_URL", None)) parser.add_argument('--aws_region', help='(string) AWS region', type=str, default=rospy.get_param("AWS_REGION", "us-east-1")) parser.add_argument('--number_of_trials', help='(integer) Number of trials', type=int, default=int(rospy.get_param("NUMBER_OF_TRIALS", 10))) parser.add_argument( '-c', '--local_model_directory', help='(string) Path to a folder containing a checkpoint \ to restore the model from.', type=str, default='./checkpoint') parser.add_argument('--number_of_resets', help='(integer) Number of resets', type=int, default=int(rospy.get_param("NUMBER_OF_RESETS", 0))) parser.add_argument('--penalty_seconds', help='(float) penalty second', type=float, default=float(rospy.get_param("PENALTY_SECONDS", 2.0))) parser.add_argument('--job_type', help='(string) job type', type=str, default=rospy.get_param("JOB_TYPE", "EVALUATION")) parser.add_argument('--is_continuous', help='(boolean) is continous after lap completion', type=bool, default=utils.str2bool( rospy.get_param("IS_CONTINUOUS", False))) parser.add_argument('--race_type', help='(string) Race type', type=str, default=rospy.get_param("RACE_TYPE", "TIME_TRIAL")) parser.add_argument('--off_track_penalty', help='(float) off track penalty second', type=float, default=float(rospy.get_param("OFF_TRACK_PENALTY", 2.0))) parser.add_argument('--collision_penalty', help='(float) collision penalty second', type=float, default=float(rospy.get_param("COLLISION_PENALTY", 5.0))) parser.add_argument('--round_robin_advance_dist', help='(float) round robin distance 0-1', type=float, default=float( rospy.get_param("ROUND_ROBIN_ADVANCE_DIST", 0.05))) parser.add_argument('--start_position_offset', help='(float) offset start 0-1', type=float, default=float( rospy.get_param("START_POSITION_OFFSET", 0.0))) args = parser.parse_args() arg_s3_bucket = args.s3_bucket arg_s3_prefix = args.s3_prefix logger.info("S3 bucket: %s \n S3 prefix: %s \n S3 endpoint URL: %s", args.s3_bucket, args.s3_prefix, args.s3_endpoint_url) metrics_s3_buckets = rospy.get_param('METRICS_S3_BUCKET') metrics_s3_object_keys = rospy.get_param('METRICS_S3_OBJECT_KEY') arg_s3_bucket, arg_s3_prefix = utils.force_list( arg_s3_bucket), utils.force_list(arg_s3_prefix) metrics_s3_buckets = utils.force_list(metrics_s3_buckets) metrics_s3_object_keys = utils.force_list(metrics_s3_object_keys) validate_list = [ arg_s3_bucket, arg_s3_prefix, metrics_s3_buckets, metrics_s3_object_keys ] simtrace_s3_bucket = rospy.get_param('SIMTRACE_S3_BUCKET', None) mp4_s3_bucket = rospy.get_param('MP4_S3_BUCKET', None) if simtrace_s3_bucket: simtrace_s3_object_prefix = rospy.get_param('SIMTRACE_S3_PREFIX') simtrace_s3_bucket = utils.force_list(simtrace_s3_bucket) simtrace_s3_object_prefix = utils.force_list(simtrace_s3_object_prefix) validate_list.extend([simtrace_s3_bucket, simtrace_s3_object_prefix]) if mp4_s3_bucket: mp4_s3_object_prefix = rospy.get_param('MP4_S3_OBJECT_PREFIX') mp4_s3_bucket = utils.force_list(mp4_s3_bucket) mp4_s3_object_prefix = utils.force_list(mp4_s3_object_prefix) validate_list.extend([mp4_s3_bucket, mp4_s3_object_prefix]) if not all([lambda x: len(x) == len(validate_list[0]), validate_list]): log_and_exit( "Eval worker error: Incorrect arguments passed: {}".format( validate_list), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) if args.number_of_resets != 0 and args.number_of_resets < MIN_RESET_COUNT: raise GenericRolloutException( "number of resets is less than {}".format(MIN_RESET_COUNT)) # Instantiate Cameras if len(arg_s3_bucket) == 1: configure_camera(namespaces=['racecar']) else: configure_camera(namespaces=[ 'racecar_{}'.format(str(agent_index)) for agent_index in range(len(arg_s3_bucket)) ]) agent_list = list() s3_bucket_dict = dict() s3_prefix_dict = dict() s3_writers = list() start_positions = get_start_positions(len(arg_s3_bucket)) done_condition = utils.str_to_done_condition( rospy.get_param("DONE_CONDITION", any)) park_positions = utils.pos_2d_str_to_list( rospy.get_param("PARK_POSITIONS", [])) # if not pass in park positions for all done condition case, use default if not park_positions: park_positions = [DEFAULT_PARK_POSITION for _ in arg_s3_bucket] for agent_index, _ in enumerate(arg_s3_bucket): agent_name = 'agent' if len(arg_s3_bucket) == 1 else 'agent_{}'.format( str(agent_index)) racecar_name = 'racecar' if len( arg_s3_bucket) == 1 else 'racecar_{}'.format(str(agent_index)) s3_bucket_dict[agent_name] = arg_s3_bucket[agent_index] s3_prefix_dict[agent_name] = arg_s3_prefix[agent_index] # download model metadata model_metadata = ModelMetadata( bucket=arg_s3_bucket[agent_index], s3_key=get_s3_key(arg_s3_prefix[agent_index], MODEL_METADATA_S3_POSTFIX), region_name=args.aws_region, s3_endpoint_url=args.s3_endpoint_url, local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format(agent_name)) _, _, version = model_metadata.get_model_metadata_info() # Select the optimal model utils.do_model_selection(s3_bucket=arg_s3_bucket[agent_index], s3_prefix=arg_s3_prefix[agent_index], region=args.aws_region, s3_endpoint_url=args.s3_endpoint_url) agent_config = { 'model_metadata': model_metadata, ConfigParams.CAR_CTRL_CONFIG.value: { ConfigParams.LINK_NAME_LIST.value: [ link_name.replace('racecar', racecar_name) for link_name in LINK_NAMES ], ConfigParams.VELOCITY_LIST.value: [ velocity_topic.replace('racecar', racecar_name) for velocity_topic in VELOCITY_TOPICS ], ConfigParams.STEERING_LIST.value: [ steering_topic.replace('racecar', racecar_name) for steering_topic in STEERING_TOPICS ], ConfigParams.CHANGE_START.value: utils.str2bool(rospy.get_param('CHANGE_START_POSITION', False)), ConfigParams.ALT_DIR.value: utils.str2bool( rospy.get_param('ALTERNATE_DRIVING_DIRECTION', False)), ConfigParams.ACTION_SPACE_PATH.value: model_metadata.local_path, ConfigParams.REWARD.value: reward_function, ConfigParams.AGENT_NAME.value: racecar_name, ConfigParams.VERSION.value: version, ConfigParams.NUMBER_OF_RESETS.value: args.number_of_resets, ConfigParams.PENALTY_SECONDS.value: args.penalty_seconds, ConfigParams.NUMBER_OF_TRIALS.value: args.number_of_trials, ConfigParams.IS_CONTINUOUS.value: args.is_continuous, ConfigParams.RACE_TYPE.value: args.race_type, ConfigParams.COLLISION_PENALTY.value: args.collision_penalty, ConfigParams.OFF_TRACK_PENALTY.value: args.off_track_penalty, ConfigParams.START_POSITION.value: start_positions[agent_index], ConfigParams.DONE_CONDITION.value: done_condition, ConfigParams.ROUND_ROBIN_ADVANCE_DIST.value: args.round_robin_advance_dist, ConfigParams.START_POSITION_OFFSET.value: args.start_position_offset } } metrics_s3_config = { MetricsS3Keys.METRICS_BUCKET.value: metrics_s3_buckets[agent_index], MetricsS3Keys.METRICS_KEY.value: metrics_s3_object_keys[agent_index], MetricsS3Keys.ENDPOINT_URL.value: rospy.get_param('S3_ENDPOINT_URL', None), # Replaced rospy.get_param('AWS_REGION') to be equal to the argument being passed # or default argument set MetricsS3Keys.REGION.value: args.aws_region } aws_region = rospy.get_param('AWS_REGION', args.aws_region) s3_writer_job_info = [] if simtrace_s3_bucket: s3_writer_job_info.append( IterationData( 'simtrace', simtrace_s3_bucket[agent_index], simtrace_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. SIM_TRACE_EVALUATION_LOCAL_FILE.value))) if mp4_s3_bucket: s3_writer_job_info.extend([ IterationData( 'pip', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_PIP_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( '45degree', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_45DEGREE_MP4_VALIDATION_LOCAL_PATH.value)), IterationData( 'topview', mp4_s3_bucket[agent_index], mp4_s3_object_prefix[agent_index], aws_region, os.path.join( ITERATION_DATA_LOCAL_FILE_PATH, agent_name, IterationDataLocalFileNames. CAMERA_TOPVIEW_MP4_VALIDATION_LOCAL_PATH.value)) ]) s3_writers.append( S3Writer(job_info=s3_writer_job_info, s3_endpoint_url=args.s3_endpoint_url)) run_phase_subject = RunPhaseSubject() agent_list.append( create_rollout_agent( agent_config, EvalMetrics(agent_name, metrics_s3_config, args.is_continuous), run_phase_subject)) agent_list.append(create_obstacles_agent()) agent_list.append(create_bot_cars_agent()) # ROS service to indicate all the robomaker markov packages are ready for consumption signal_robomaker_markov_package_ready() PhaseObserver('/agent/training_phase', run_phase_subject) enable_domain_randomization = utils.str2bool( rospy.get_param('ENABLE_DOMAIN_RANDOMIZATION', False)) sm_hyperparams_dict = {} graph_manager, _ = get_graph_manager( hp_dict=sm_hyperparams_dict, agent_list=agent_list, run_phase_subject=run_phase_subject, enable_domain_randomization=enable_domain_randomization, done_condition=done_condition) ds_params_instance = S3BotoDataStoreParameters( aws_region=args.aws_region, bucket_names=s3_bucket_dict, base_checkpoint_dir=args.local_model_directory, s3_folders=s3_prefix_dict, s3_endpoint_url=args.s3_endpoint_url) graph_manager.data_store = S3BotoDataStore(params=ds_params_instance, graph_manager=graph_manager, ignore_lock=True) graph_manager.env_params.seed = 0 task_parameters = TaskParameters() task_parameters.checkpoint_restore_path = args.local_model_directory evaluation_worker(graph_manager=graph_manager, number_of_trials=args.number_of_trials, task_parameters=task_parameters, s3_writers=s3_writers, is_continuous=args.is_continuous, park_positions=park_positions)
def __init__(self, racecar_names): ''' Constructor for the Deep Racer object, will load track and waypoints ''' # Wait for required services to be available rospy.wait_for_service(SET_MODEL_STATE) rospy.wait_for_service(GazeboServiceName.PAUSE_PHYSICS.value) rospy.wait_for_service(GazeboServiceName.GET_MODEL_PROPERTIES.value) rospy.wait_for_service(GazeboServiceName.GET_VISUAL_NAMES.value) rospy.wait_for_service(GazeboServiceName.GET_VISUALS.value) rospy.wait_for_service(GazeboServiceName.SET_VISUAL_COLORS.value) rospy.wait_for_service(GazeboServiceName.SET_VISUAL_TRANSPARENCIES.value) rospy.wait_for_service(GazeboServiceName.SET_VISUAL_VISIBLES.value) self.racer_num = len(racecar_names) for racecar_name in racecar_names: wait_for_model(model_name=racecar_name, relative_entity_name='') self.car_colors = force_list(rospy.get_param(const.YamlKey.CAR_COLOR.value, [const.CarColorType.BLACK.value] * len(racecar_names))) self.shell_types = force_list(rospy.get_param(const.YamlKey.BODY_SHELL_TYPE.value, [const.BodyShellType.DEFAULT.value] * len(racecar_names))) # Gazebo service that allows us to position the car self.model_state_client = ServiceProxyWrapper(SET_MODEL_STATE, SetModelState) self.get_model_prop = ServiceProxyWrapper(GazeboServiceName.GET_MODEL_PROPERTIES.value, GetModelProperties) self.get_visual_names = ServiceProxyWrapper(GazeboServiceName.GET_VISUAL_NAMES.value, GetVisualNames) self.get_visuals = ServiceProxyWrapper(GazeboServiceName.GET_VISUALS.value, GetVisuals) self.set_visual_colors = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_COLORS.value, SetVisualColors) self.set_visual_transparencies = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_TRANSPARENCIES.value, SetVisualTransparencies) self.set_visual_visibles = ServiceProxyWrapper(GazeboServiceName.SET_VISUAL_VISIBLES.value, SetVisualVisibles) # Place the car at the starting point facing the forward direction # Instantiate cameras main_cameras, sub_camera = configure_camera(namespaces=racecar_names) [camera.detach() for camera in main_cameras.values()] sub_camera.detach() # Get the root directory of the ros package, this will contain the models deepracer_path = rospkg.RosPack().get_path("deepracer_simulation_environment") # Grab the track data self.track_data = TrackData.get_instance() # Set all racers start position in track data self.start_positions = get_start_positions(self.racer_num) car_poses = [] for racecar_idx, racecar_name in enumerate(racecar_names): car_model_state = self.get_initial_position(racecar_name, racecar_idx) car_poses.append(car_model_state.pose) self.update_model_visual(racecar_name, self.shell_types[racecar_idx], self.car_colors[racecar_idx]) # Let KVS collect a few frames before pausing the physics, so the car # will appear on the track time.sleep(1) pause_physics = ServiceProxyWrapper('/gazebo/pause_physics', Empty) logger.info("Pausing physics after initializing the cars") pause_physics(EmptyRequest()) for racecar_name, car_pose in zip(racecar_names, car_poses): main_cameras[racecar_name].spawn_model(car_pose, os.path.join(deepracer_path, "models", "camera", "model.sdf")) logger.info("Spawning sub camera model") # Spawn the top camera model sub_camera.spawn_model(None, os.path.join(deepracer_path, "models", "top_camera", "model.sdf"))
def main(): """ Main function for downloading yaml params """ # parse argument s3_region = sys.argv[1] s3_bucket = sys.argv[2] s3_prefix = sys.argv[3] s3_yaml_name = sys.argv[4] launch_name = sys.argv[5] yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name)) try: # create boto3 session/client and download yaml/json file session = boto3.session.Session() s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None) if s3_endpoint_url is not None: LOG.info('Endpoint URL {}'.format(s3_endpoint_url)) rospy.set_param('S3_ENDPOINT_URL', s3_endpoint_url) s3_client = session.client('s3', region_name=s3_region, endpoint_url=s3_endpoint_url, config=get_boto_config()) local_yaml_path = os.path.abspath( os.path.join(os.getcwd(), s3_yaml_name)) s3_client.download_file(Bucket=s3_bucket, Key=yaml_key, Filename=local_yaml_path) # Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing default_yaml_values = { RACE_TYPE_YAML_KEY: TIME_TRIAL_RACE_TYPE, MODEL_S3_BUCKET_YAML_KEY: s3_bucket, MODEL_S3_PREFIX_YAML_KEY: s3_prefix, CAR_COLOR_YAML_KEY: DEFAULT_COLOR, BODY_SHELL_TYPE_YAML_KEY: None, MODEL_METADATA_FILE_S3_YAML_KEY: None, RACER_NAME_YAML_KEY: None } yaml_dict = get_yaml_dict(local_yaml_path) yaml_values = get_yaml_values(yaml_dict, default_yaml_values) # Forcing the yaml parameter to list force_list_params = [ MODEL_METADATA_FILE_S3_YAML_KEY, MODEL_S3_BUCKET_YAML_KEY, MODEL_S3_PREFIX_YAML_KEY, CAR_COLOR_YAML_KEY, BODY_SHELL_TYPE_YAML_KEY, RACER_NAME_YAML_KEY ] for params in force_list_params: yaml_values[params] = force_list(yaml_values[params]) # Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats if None in yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY]: # MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation # Assume model_metadata.json is present in the s3_prefix/model/ folder yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY] = list() for s3_prefix in yaml_values[MODEL_S3_PREFIX_YAML_KEY]: yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY].append( os.path.join(s3_prefix, 'model/model_metadata.json')) # Set multicar value if there is more than one value in MODEL_S3_BUCKET_YAML_KEY. multicar = len(yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 # Set f1 as true if RACE_TYPE is F1 is_f1 = yaml_values[RACE_TYPE_YAML_KEY] == F1_RACE_TYPE # Validate the yaml values validate_yaml_values(yaml_values, multicar) # List of racecar names that should include second camera while launching racecars_with_stereo_cameras = list() # List of racecar names that should include lidar while launching racecars_with_lidars = list() # List of SimApp versions simapp_versions = list() # List of body shell types body_shell_types = yaml_values[BODY_SHELL_TYPE_YAML_KEY] racer_names = yaml_values[RACER_NAME_YAML_KEY] if None in body_shell_types: # use default shells if None in racer_names: body_shell_types = [BodyShellType.DEFAULT.value] * len( yaml_values[MODEL_S3_BUCKET_YAML_KEY]) # use default shells for regular user and f1 shell for users in F1_SHELL_USERS_LIST else: body_shell_types = [ BodyShellType.F1_2021.value if racer_alias in F1_SHELL_USERS_LIST else BodyShellType.DEFAULT.value for racer_alias in yaml_values[RACER_NAME_YAML_KEY] ] yaml_dict[BODY_SHELL_TYPE_YAML_KEY] = body_shell_types # override local yaml file with updated BODY_SHELL_TYPE with open(local_yaml_path, 'w') as yaml_file: yaml.dump(yaml_dict, yaml_file) for agent_index, model_s3_bucket in enumerate( yaml_values[MODEL_S3_BUCKET_YAML_KEY]): racecar_name = 'racecar_' + str(agent_index) if len( yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 else 'racecar' # Make a local folder with the racecar name to download the model_metadata.json if not os.path.exists(os.path.join(os.getcwd(), racecar_name)): os.makedirs(os.path.join(os.getcwd(), racecar_name)) local_model_metadata_path = os.path.abspath( os.path.join(os.path.join(os.getcwd(), racecar_name), 'model_metadata.json')) json_key = yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY][ agent_index] json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '') s3_client.download_file(Bucket=model_s3_bucket, Key=json_key, Filename=local_model_metadata_path) sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata( local_model_metadata_path) simapp_versions.append(simapp_version) if Input.STEREO.value in sensors: racecars_with_stereo_cameras.append(racecar_name) if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors: racecars_with_lidars.append(racecar_name) cmd = [ ''.join(("roslaunch deepracer_simulation_environment {} ".format( launch_name), "local_yaml_path:={} ".format(local_yaml_path), "racecars_with_stereo_cameras:={} ".format( ','.join(racecars_with_stereo_cameras)), "racecars_with_lidars:={} ".format( ','.join(racecars_with_lidars)), "multicar:={} ".format(multicar), "body_shell_types:={} ".format( ','.join(body_shell_types)), "simapp_versions:={} ".format(','.join(simapp_versions)), "f1:={}".format(is_f1))) ] Popen(cmd, shell=True, executable="/bin/bash") except botocore.exceptions.ClientError as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400) except botocore.exceptions.EndpointConnectionError: log_and_exit("No Internet connection or s3 service unavailable", SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) except Exception as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)