def write_metrics_to_s3(bucket, key, region, metrics): '''Helper method that uploads the desired metrics to s3 bucket - String with S3 bucket where metrics should be written key - String with S3 bucket key where metrics should be written region - String with aws region metrics - Dictionary with metrics to write to s3 ''' try: s3_extra_args = get_s3_kms_extra_args() session = boto3.session.Session() s3_client = session.client('s3', region_name=region, config=get_boto_config()) s3_client.put_object(Bucket=bucket, Key=key, Body=bytes(json.dumps(metrics), encoding='utf-8'), **s3_extra_args) except botocore.exceptions.ClientError as err: log_and_exit( "Unable to write metrics to s3: bucket: {}, error: {}".format( bucket, err.response['Error']['Code']), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400) except Exception as ex: log_and_exit("Unable to write metrics to s3, exception: {}".format(ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def upload_mpu_part_to_s3(self, episodes): if self.data_state != SIMTRACE_DATA_UPLOAD_UNKNOWN_STATE: logger.debug( "simtrace_data: Uploading mpu_part_to_s3::: mpu_id-{} mpu_part_number-{} episode-{}" .format(self.mpu_id, self.mpu_part_number, episodes)) self.mpu_episodes = episodes s3_client = boto3.session.Session().client( 's3', region_name=self.aws_region, config=utils.get_boto_config()) metrics_body = self.simtrace_csv_data.getvalue() part = s3_client.upload_part(Body=bytes(metrics_body, encoding='utf-8'), Bucket=self.s3_bucket, Key=self.s3_object_key, UploadId=self.mpu_id, PartNumber=self.mpu_part_number) self.mpu_parts.append({ "PartNumber": self.mpu_part_number, "ETag": part["ETag"] }) self.mpu_part_number += 1 logger.info( "simtrace_data: Uploaded mpu_part_to_s3::: done! episode-{} mpu_id-{} mpu_part_number-{} mpu_parts-{}" .format(episodes, self.mpu_id, self.mpu_part_number, self.mpu_parts))
def setup_mutipart_upload(self): logger.info("simtrace_data: setup_mutipart_upload to %s", self.s3_bucket) #setup for SIM_TRACE data incremental uploads to S3 self.simtrace_csv_data = StringIO() self.csvwriter = csv.writer(self.simtrace_csv_data) self.csvwriter.writerow(SIMTRACE_CSV_DATA_HEADER) self.aws_region = rospy.get_param('AWS_REGION') logger.info( "simtrace_data: setup_mutipart_upload on s3_bucket {} s3_object_key {} region {}" .format(self.s3_bucket, self.s3_object_key, self.aws_region)) #initiate the multipart upload s3_client = boto3.session.Session().client( 's3', region_name=self.aws_region, config=utils.get_boto_config()) self.mpu = s3_client.create_multipart_upload( Bucket=self.s3_bucket, Key=self.s3_object_key) self.mpu_id = self.mpu["UploadId"] self.mpu_part_number = 1 self.mpu_parts = [] self.mpu_episodes = 0 self.total_upload_size = 0 self.data_state = SIMTRACE_DATA_UPLOAD_INIT_DONE logger.info( "simtrace_data: setup_mutipart_upload done! mpu_id= %s mpu_part_number", self.mpu_id)
def get_client(self): session = boto3.session.Session() return session.client('s3', region_name=self.aws_region, endpoint_url=self.s3_endpoint_url, config=get_boto_config())
def test_get_boto_config(): """This function checks the functionality of get_boto_config function in markov/utils.py """ utils.get_boto_config()
def main(): """ Main function for tournament""" try: # parse argument s3_region = sys.argv[1] s3_bucket = sys.argv[2] s3_prefix = sys.argv[3] s3_yaml_name = sys.argv[4] # create boto3 session/client and download yaml/json file session = boto3.session.Session() s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None) s3_client = session.client('s3', region_name=s3_region, endpoint_url=s3_endpoint_url, config=get_boto_config()) yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name)) local_yaml_path = os.path.abspath( os.path.join(os.getcwd(), s3_yaml_name)) s3_client.download_file(Bucket=s3_bucket, Key=yaml_key, Filename=local_yaml_path) # Intermediate tournament files queue_pickle_name = 'tournament_candidate_queue.pkl' queue_pickle_s3_key = os.path.normpath( os.path.join(s3_prefix, queue_pickle_name)) local_queue_pickle_path = os.path.abspath( os.path.join(os.getcwd(), queue_pickle_name)) report_pickle_name = 'tournament_report.pkl' report_pickle_s3_key = os.path.normpath( os.path.join(s3_prefix, report_pickle_name)) local_report_pickle_path = os.path.abspath( os.path.join(os.getcwd(), report_pickle_name)) final_report_name = 'tournament_report.json' final_report_s3_key = os.path.normpath( os.path.join(s3_prefix, final_report_name)) try: s3_client.download_file(Bucket=s3_bucket, Key=queue_pickle_s3_key, Filename=local_queue_pickle_path) s3_client.download_file(Bucket=s3_bucket, Key=report_pickle_s3_key, Filename=local_report_pickle_path) except: pass # Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing yaml_dict = get_yaml_dict(local_yaml_path) # Forcing the yaml parameter to list force_list_params = [ MODEL_S3_BUCKET_YAML_KEY, MODEL_S3_PREFIX_YAML_KEY, MODEL_METADATA_FILE_S3_YAML_KEY, METRICS_S3_BUCKET_YAML_KEY, METRICS_S3_PREFIX_YAML_KEY, SIMTRACE_S3_BUCKET_YAML_KEY, SIMTRACE_S3_PREFIX_YAML_KEY, MP4_S3_BUCKET_YAML_KEY, MP4_S3_PREFIX_YAML_KEY, DISPLAY_NAME_YAML_KEY ] for params in force_list_params: yaml_dict[params] = force_list(yaml_dict.get(params, None)) # Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats if None in yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY]: # MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation # Assume model_metadata.json is present in the s3_prefix/model/ folder yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY] = list() for s3_prefix in yaml_dict[MODEL_S3_PREFIX_YAML_KEY]: yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY].append( os.path.join(s3_prefix, 'model/model_metadata.json')) # Validate the yaml values validate_yaml_values(yaml_dict) if os.path.exists(local_queue_pickle_path): with open(local_queue_pickle_path, 'rb') as f: tournament_candidate_queue = pickle.load(f) with open(local_report_pickle_path, 'rb') as f: tournament_report = pickle.load(f) logger.info('tournament_candidate_queue loaded from existing file') else: logger.info('tournament_candidate_queue initialized') tournament_candidate_queue = deque() for agent_idx, _ in enumerate(yaml_dict[MODEL_S3_BUCKET_YAML_KEY]): tournament_candidate_queue.append( (yaml_dict[MODEL_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[MODEL_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[MODEL_METADATA_FILE_S3_YAML_KEY][agent_idx], yaml_dict[METRICS_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[METRICS_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[SIMTRACE_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[SIMTRACE_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[MP4_S3_BUCKET_YAML_KEY][agent_idx], yaml_dict[MP4_S3_PREFIX_YAML_KEY][agent_idx], yaml_dict[DISPLAY_NAME_YAML_KEY][agent_idx])) tournament_report = [] race_idx = len(tournament_report) while len(tournament_candidate_queue) > 1: car1 = tournament_candidate_queue.popleft() car2 = tournament_candidate_queue.popleft() (car1_model_s3_bucket, car1_s3_prefix, car1_model_metadata, car1_metrics_bucket, car1_metrics_s3_key, car1_simtrace_bucket, car1_simtrace_prefix, car1_mp4_bucket, car1_mp4_prefix, car1_display_name) = car1 (car2_model_s3_bucket, car2_s3_prefix, car2_model_metadata, car2_metrics_bucket, car2_metrics_s3_key, car2_simtrace_bucket, car2_simtrace_prefix, car2_mp4_bucket, car2_mp4_prefix, car2_display_name) = car2 race_yaml_dict = generate_race_yaml(yaml_dict=yaml_dict, car1=car1, car2=car2, race_idx=race_idx) race_car_colors = ["Orange", "Purple"] race_model_s3_buckets = [ car1_model_s3_bucket, car2_model_s3_bucket ] race_model_metadatas = [car1_model_metadata, car2_model_metadata] # List of directories created dirs_to_delete = list() yaml_dir = os.path.abspath(os.path.join(os.getcwd(), str(race_idx))) os.makedirs(yaml_dir) dirs_to_delete.append(yaml_dir) race_yaml_path = os.path.abspath( os.path.join(yaml_dir, 'evaluation_params.yaml')) with open(race_yaml_path, 'w') as race_yaml_file: yaml.dump(race_yaml_dict, race_yaml_file) # List of racecar names that should include second camera while launching racecars_with_stereo_cameras = list() # List of racecar names that should include lidar while launching racecars_with_lidars = list() # List of SimApp versions simapp_versions = list() for agent_index, model_s3_bucket in enumerate( race_model_s3_buckets): racecar_name = 'racecar_' + str(agent_index) # Make a local folder with the racecar name to download the model_metadata.json os.makedirs(os.path.join(os.getcwd(), racecar_name)) dirs_to_delete.append(os.path.join(os.getcwd(), racecar_name)) local_model_metadata_path = os.path.abspath( os.path.join(os.path.join(os.getcwd(), racecar_name), 'model_metadata.json')) json_key = race_model_metadatas[agent_index] json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '') s3_client.download_file(Bucket=model_s3_bucket, Key=json_key, Filename=local_model_metadata_path) sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata( local_model_metadata_path) simapp_versions.append(simapp_version) if Input.STEREO.value in sensors: racecars_with_stereo_cameras.append(racecar_name) if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors: racecars_with_lidars.append(racecar_name) cmd = [ os.path.join(os.path.dirname(os.path.abspath(__file__)), "tournament_race_node.py"), str(race_idx), race_yaml_path, ','.join(racecars_with_stereo_cameras), ','.join(racecars_with_lidars), ','.join(race_car_colors), ','.join(simapp_versions) ] try: return_code, _, stderr = run_cmd(cmd_args=cmd, shell=False, stdout=None, stderr=None) except KeyboardInterrupt: logger.info( "KeyboardInterrupt raised, SimApp must be faulted! exiting..." ) return # Retrieve winner and append tournament report with open('race_report.pkl', 'rb') as f: race_report = pickle.load(f) race_report['race_idx'] = race_idx winner = car1 if race_report[ 'winner'] == car1_display_name else car2 logger.info("race {}'s winner: {}".format(race_idx, race_report['winner'])) tournament_candidate_queue.append(winner) tournament_report.append(race_report) # Clean up directories created for dir_to_delete in dirs_to_delete: shutil.rmtree(dir_to_delete, ignore_errors=True) race_idx += 1 # Persist latest queue and report to use after job restarts. with open(local_queue_pickle_path, 'wb') as f: pickle.dump(tournament_candidate_queue, f, protocol=2) s3_client.upload_file(Filename=local_queue_pickle_path, Bucket=s3_bucket, Key=queue_pickle_s3_key) with open(local_report_pickle_path, 'wb') as f: pickle.dump(tournament_report, f, protocol=2) s3_client.upload_file(Filename=local_report_pickle_path, Bucket=s3_bucket, Key=report_pickle_s3_key) # If there is more than 1 candidates then restart the simulation job otherwise # tournament is finished, persists final report and ends the job. if len(tournament_candidate_queue) > 1: restart_simulation_job( os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'), s3_region) break else: # Persist final tournament report in json format # and terminate the job by canceling it s3_client.put_object(Bucket=s3_bucket, Key=final_report_s3_key, Body=json.dumps(tournament_report)) cancel_simulation_job( os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'), s3_region) except Exception as e: log_and_exit( "Tournament node failed: s3_bucket: {}, yaml_key: {}, {}".format( s3_bucket, yaml_key, e), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def complete_upload_to_s3(self): logger.info( "simtrace_data: complete_upload_to_s3::: data_state-{}".format( self.data_state)) try: if self.data_state == SIMTRACE_DATA_MPU_UPLOAD_IN_PROGRESS: #Multi-part upload to s3 self.data_state = SIMTRACE_DATA_UPLOAD_DONE logger.info( "simtrace_data: complete_upload_to_s3::Multi-part upload to S3 in progress, upload the last part number-{}, then complete mpu" .format(self.mpu_part_number)) self.upload_mpu_part_to_s3(self.mpu_episodes) #now complete the multi-part-upload session = boto3.session.Session() s3_client = session.client('s3', region_name=self.aws_region, config=utils.get_boto_config()) result = s3_client.complete_multipart_upload( Bucket=self.s3_bucket, Key=self.s3_object_key, UploadId=self.mpu_id, MultipartUpload={"Parts": self.mpu_parts}) self.data_state = SIMTRACE_DATA_UPLOAD_DONE logger.info( "simtrace_data: complete_upload_to_s3 ::: multi-part-upload done,total raw size={}bytes result={}" .format(self.total_upload_size, result)) else: #One-time upload to s3 if self.data_state == SIMTRACE_DATA_UPLOAD_INIT_DONE and self.data_state != SIMTRACE_DATA_UPLOAD_DONE: self.data_state = SIMTRACE_DATA_UPLOAD_DONE logger.info( "simtrace_data: complete_upload_to_s3 ::: write simtrace data to s3" ) session = boto3.session.Session() s3_client = session.client( 's3', region_name=self.aws_region, config=utils.get_boto_config()) # cancel multipart upload process logger.info( "simtrace_data: multi-part upload not required, cancel it before uploading the complete S3 object" ) s3_client.abort_multipart_upload( Bucket=self.s3_bucket, Key=self.s3_object_key, UploadId=self.mpu_id) metrics_body = self.simtrace_csv_data.getvalue() logger.info( "simtrace_data: complete_upload_to_s3:: write to s3 csv-formatted-data size={}bytes" .format(sys.getsizeof(metrics_body))) result = s3_client.put_object(Bucket=self.s3_bucket, Key=self.s3_object_key, Body=bytes( metrics_body, encoding='utf-8')) logger.info( "simtrace_data: complete_upload_to_s3:: done writing simtrace total-unformatted-data size={}bytes to s3. result{}" .format(self.total_upload_size, result)) self.reset_mpu_part(self.mpu_episodes) except Exception as e: logger.info( "simtrace_data: complete_upload_to_s3:: exception-{} ". format(e))
def main(): """ Main function for downloading yaml params """ try: # parse argument s3_region = sys.argv[1] s3_bucket = sys.argv[2] s3_prefix = sys.argv[3] s3_yaml_name = sys.argv[4] launch_name = sys.argv[5] # create boto3 session/client and download yaml/json file session = boto3.session.Session() s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None) if s3_endpoint_url is not None: LOG.info('Endpoint URL {}'.format(s3_endpoint_url)) rospy.set_param('S3_ENDPOINT_URL', s3_endpoint_url) else: # create boto3 session/client and download yaml/json file ec2_client = session.client('ec2', s3_region) LOG.info('Checking internet connection...') response = ec2_client.describe_vpcs() if not response['Vpcs']: log_and_exit("No VPC attached to instance", SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) LOG.info('Verified internet connection') s3_client = session.client('s3', region_name=s3_region, endpoint_url=s3_endpoint_url, config=get_boto_config()) yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name)) local_yaml_path = os.path.abspath( os.path.join(os.getcwd(), s3_yaml_name)) s3_client.download_file(Bucket=s3_bucket, Key=yaml_key, Filename=local_yaml_path) # Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing default_yaml_values = { RACE_TYPE_YAML_KEY: TIME_TRIAL_RACE_TYPE, MODEL_S3_BUCKET_YAML_KEY: s3_bucket, MODEL_S3_PREFIX_YAML_KEY: s3_prefix, CAR_COLOR_YAML_KEY: DEFAULT_COLOR, MODEL_METADATA_FILE_S3_YAML_KEY: None } yaml_dict = get_yaml_dict(local_yaml_path) yaml_values = get_yaml_values(yaml_dict, default_yaml_values) # Forcing the yaml parameter to list force_list_params = [ MODEL_METADATA_FILE_S3_YAML_KEY, MODEL_S3_BUCKET_YAML_KEY, MODEL_S3_PREFIX_YAML_KEY, CAR_COLOR_YAML_KEY ] for params in force_list_params: yaml_values[params] = force_list(yaml_values[params]) # Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats if None in yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY]: # MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation # Assume model_metadata.json is present in the s3_prefix/model/ folder yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY] = list() for s3_prefix in yaml_values[MODEL_S3_PREFIX_YAML_KEY]: yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY].append( os.path.join(s3_prefix, 'model/model_metadata.json')) # Set multicar value if its a head to model racetype multicar = yaml_values[RACE_TYPE_YAML_KEY] == HEAD_TO_MODEL_RACE_TYPE # Validate the yaml values validate_yaml_values(yaml_values, multicar) # List of racecar names that should include second camera while launching racecars_with_stereo_cameras = list() # List of racecar names that should include lidar while launching racecars_with_lidars = list() # List of SimApp versions simapp_versions = list() for agent_index, model_s3_bucket in enumerate( yaml_values[MODEL_S3_BUCKET_YAML_KEY]): racecar_name = 'racecar_' + str(agent_index) if len( yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 else 'racecar' # Make a local folder with the racecar name to download the model_metadata.json if not os.path.exists(os.path.join(os.getcwd(), racecar_name)): os.makedirs(os.path.join(os.getcwd(), racecar_name)) local_model_metadata_path = os.path.abspath( os.path.join(os.path.join(os.getcwd(), racecar_name), 'model_metadata.json')) json_key = yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY][ agent_index] json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '') s3_client.download_file(Bucket=model_s3_bucket, Key=json_key, Filename=local_model_metadata_path) sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata( local_model_metadata_path) simapp_versions.append(simapp_version) if Input.STEREO.value in sensors: racecars_with_stereo_cameras.append(racecar_name) if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors: racecars_with_lidars.append(racecar_name) cmd = [ ''.join(("roslaunch deepracer_simulation_environment {} ".format( launch_name), "local_yaml_path:={} ".format(local_yaml_path), "racecars_with_stereo_cameras:={} ".format( ','.join(racecars_with_stereo_cameras)), "racecars_with_lidars:={} multicar:={} ".format( ','.join(racecars_with_lidars), multicar), "car_colors:={} simapp_versions:={}".format( ','.join(yaml_values[CAR_COLOR_YAML_KEY]), ','.join(simapp_versions)))) ] Popen(cmd, shell=True, executable="/bin/bash") except botocore.exceptions.ClientError as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400) except botocore.exceptions.EndpointConnectionError: log_and_exit("No Internet connection or s3 service unavailable", SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) except Exception as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def _get_client(self): session = boto3.session.Session() return session.client('s3', region_name=self.params.aws_region, config=get_boto_config())
def main(): """ Main function for tournament""" try: # parse argument s3_region = sys.argv[1] s3_bucket = sys.argv[2] s3_prefix = sys.argv[3] s3_yaml_name = sys.argv[4] # create boto3 session/client and download yaml/json file session = boto3.session.Session() s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None) s3_client = session.client('s3', region_name=s3_region, endpoint_url=s3_endpoint_url, config=get_boto_config()) # Intermediate tournament files queue_pickle_name = 'tournament_candidate_queue.pkl' queue_pickle_s3_key = os.path.normpath( os.path.join(s3_prefix, queue_pickle_name)) local_queue_pickle_path = os.path.abspath( os.path.join(os.getcwd(), queue_pickle_name)) report_pickle_name = 'tournament_report.pkl' report_pickle_s3_key = os.path.normpath( os.path.join(s3_prefix, report_pickle_name)) local_report_pickle_path = os.path.abspath( os.path.join(os.getcwd(), report_pickle_name)) final_report_name = 'tournament_report.json' final_report_s3_key = os.path.normpath( os.path.join(s3_prefix, final_report_name)) try: s3_client.download_file(Bucket=s3_bucket, Key=queue_pickle_s3_key, Filename=local_queue_pickle_path) s3_client.download_file(Bucket=s3_bucket, Key=report_pickle_s3_key, Filename=local_report_pickle_path) except: pass # download yaml file yaml_file = YamlFile( agent_type=AgentType.TOURNAMENT.value, bucket=s3_bucket, s3_key=get_s3_key(s3_prefix, s3_yaml_name), region_name=s3_region, s3_endpoint_url=s3_endpoint_url, local_path=YAML_LOCAL_PATH_FORMAT.format(s3_yaml_name)) yaml_dict = yaml_file.get_yaml_values() if os.path.exists(local_queue_pickle_path): with open(local_queue_pickle_path, 'rb') as f: tournament_candidate_queue = pickle.load(f) with open(local_report_pickle_path, 'rb') as f: tournament_report = pickle.load(f) logger.info('tournament_candidate_queue loaded from existing file') else: logger.info('tournament_candidate_queue initialized') tournament_candidate_queue = deque() for agent_idx, _ in enumerate( yaml_dict[YamlKey.MODEL_S3_BUCKET_YAML_KEY.value]): tournament_candidate_queue.append(( yaml_dict[YamlKey.MODEL_S3_BUCKET_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.MODEL_S3_PREFIX_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.MODEL_METADATA_FILE_S3_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.METRICS_S3_BUCKET_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.METRICS_S3_PREFIX_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.SIMTRACE_S3_BUCKET_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.SIMTRACE_S3_PREFIX_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.MP4_S3_BUCKET_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.MP4_S3_PREFIX_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.DISPLAY_NAME_YAML_KEY.value][agent_idx], # TODO: Deprecate the DISPLAY_NAME and use only the RACER_NAME without if else check "" if None in yaml_dict.get(YamlKey.RACER_NAME_YAML_KEY.value, [None]) \ else yaml_dict[YamlKey.RACER_NAME_YAML_KEY.value][agent_idx], yaml_dict[YamlKey.BODY_SHELL_TYPE_YAML_KEY.value][agent_idx] )) tournament_report = {"race_results": []} race_idx = len(tournament_report["race_results"]) while len(tournament_candidate_queue) > 1: car1 = tournament_candidate_queue.popleft() car2 = tournament_candidate_queue.popleft() (car1_model_s3_bucket, car1_s3_prefix, car1_model_metadata, car1_metrics_bucket, car1_metrics_s3_key, car1_simtrace_bucket, car1_simtrace_prefix, car1_mp4_bucket, car1_mp4_prefix, car1_display_name, car1_racer_name, car1_body_shell_type) = car1 (car2_model_s3_bucket, car2_s3_prefix, car2_model_metadata, car2_metrics_bucket, car2_metrics_s3_key, car2_simtrace_bucket, car2_simtrace_prefix, car2_mp4_bucket, car2_mp4_prefix, car2_display_name, car2_racer_name, car2_body_shell_type) = car2 race_yaml_dict = generate_race_yaml(yaml_dict=yaml_dict, car1=car1, car2=car2, race_idx=race_idx) race_model_s3_buckets = [ car1_model_s3_bucket, car2_model_s3_bucket ] race_model_metadatas = [car1_model_metadata, car2_model_metadata] body_shell_types = [car1_body_shell_type, car2_body_shell_type] # List of directories created dirs_to_delete = list() yaml_dir = os.path.abspath(os.path.join(os.getcwd(), str(race_idx))) os.makedirs(yaml_dir) dirs_to_delete.append(yaml_dir) race_yaml_path = os.path.abspath( os.path.join(yaml_dir, 'evaluation_params.yaml')) with open(race_yaml_path, 'w') as race_yaml_file: yaml.dump(race_yaml_dict, race_yaml_file) # List of racecar names that should include second camera while launching racecars_with_stereo_cameras = list() # List of racecar names that should include lidar while launching racecars_with_lidars = list() # List of SimApp versions simapp_versions = list() for agent_index, model_s3_bucket in enumerate( race_model_s3_buckets): racecar_name = 'racecar_' + str(agent_index) json_key = race_model_metadatas[agent_index] # download model metadata try: model_metadata = ModelMetadata( bucket=model_s3_bucket, s3_key=json_key, region_name=s3_region, s3_endpoint_url=s3_endpoint_url, local_path=MODEL_METADATA_LOCAL_PATH_FORMAT.format( racecar_name)) dirs_to_delete.append(model_metadata.local_dir) except Exception as e: log_and_exit( "Failed to download model_metadata file: s3_bucket: {}, s3_key: {}, {}" .format(model_s3_bucket, json_key, e), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) sensors, _, simapp_version = model_metadata.get_model_metadata_info( ) simapp_versions.append(str(simapp_version)) if Input.STEREO.value in sensors: racecars_with_stereo_cameras.append(racecar_name) if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors: racecars_with_lidars.append(racecar_name) cmd = [ os.path.join(os.path.dirname(os.path.abspath(__file__)), "tournament_race_node.py"), str(race_idx), race_yaml_path, ','.join(racecars_with_stereo_cameras), ','.join(racecars_with_lidars), ','.join(simapp_versions), ','.join(body_shell_types) ] try: return_code, _, stderr = run_cmd(cmd_args=cmd, shell=False, stdout=None, stderr=None) except KeyboardInterrupt: logger.info( "KeyboardInterrupt raised, SimApp must be faulted! exiting..." ) return # Retrieve winner and append tournament report with open('race_report.pkl', 'rb') as f: race_report = pickle.load(f) race_report['race_idx'] = race_idx winner = car1 if race_report[ 'winner'] == car1_display_name else car2 logger.info("race {}'s winner: {}".format(race_idx, race_report['winner'])) tournament_candidate_queue.append(winner) tournament_report["race_results"].append(race_report) # Clean up directories created for dir_to_delete in dirs_to_delete: shutil.rmtree(dir_to_delete, ignore_errors=True) race_idx += 1 s3_extra_args = get_s3_kms_extra_args() # Persist latest queue and report to use after job restarts. with open(local_queue_pickle_path, 'wb') as f: pickle.dump(tournament_candidate_queue, f, protocol=2) s3_client.upload_file(Filename=local_queue_pickle_path, Bucket=s3_bucket, Key=queue_pickle_s3_key, ExtraArgs=s3_extra_args) with open(local_report_pickle_path, 'wb') as f: pickle.dump(tournament_report, f, protocol=2) s3_client.upload_file(Filename=local_report_pickle_path, Bucket=s3_bucket, Key=report_pickle_s3_key, ExtraArgs=s3_extra_args) # If there is more than 1 candidates then restart the simulation job otherwise # tournament is finished, persists final report and ends the job. if len(tournament_candidate_queue) > 1: restart_simulation_job( os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'), s3_region) break else: # Persist final tournament report in json format # and terminate the job by canceling it s3_client.put_object(Bucket=s3_bucket, Key=final_report_s3_key, Body=json.dumps(tournament_report), **s3_extra_args) cancel_simulation_job( os.environ.get('AWS_ROBOMAKER_SIMULATION_JOB_ARN'), s3_region) except ValueError as ex: log_and_exit("User modified model_metadata.json: {}".format(ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400) except Exception as e: log_and_exit("Tournament node failed: {}".format(e), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)
def main(): """ Main function for downloading yaml params """ # parse argument s3_region = sys.argv[1] s3_bucket = sys.argv[2] s3_prefix = sys.argv[3] s3_yaml_name = sys.argv[4] launch_name = sys.argv[5] yaml_key = os.path.normpath(os.path.join(s3_prefix, s3_yaml_name)) try: # create boto3 session/client and download yaml/json file session = boto3.session.Session() s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL", None) if s3_endpoint_url is not None: LOG.info('Endpoint URL {}'.format(s3_endpoint_url)) rospy.set_param('S3_ENDPOINT_URL', s3_endpoint_url) s3_client = session.client('s3', region_name=s3_region, endpoint_url=s3_endpoint_url, config=get_boto_config()) local_yaml_path = os.path.abspath( os.path.join(os.getcwd(), s3_yaml_name)) s3_client.download_file(Bucket=s3_bucket, Key=yaml_key, Filename=local_yaml_path) # Get values passed in yaml files. Default values are for backward compatibility and for single racecar racing default_yaml_values = { RACE_TYPE_YAML_KEY: TIME_TRIAL_RACE_TYPE, MODEL_S3_BUCKET_YAML_KEY: s3_bucket, MODEL_S3_PREFIX_YAML_KEY: s3_prefix, CAR_COLOR_YAML_KEY: DEFAULT_COLOR, BODY_SHELL_TYPE_YAML_KEY: None, MODEL_METADATA_FILE_S3_YAML_KEY: None, RACER_NAME_YAML_KEY: None } yaml_dict = get_yaml_dict(local_yaml_path) yaml_values = get_yaml_values(yaml_dict, default_yaml_values) # Forcing the yaml parameter to list force_list_params = [ MODEL_METADATA_FILE_S3_YAML_KEY, MODEL_S3_BUCKET_YAML_KEY, MODEL_S3_PREFIX_YAML_KEY, CAR_COLOR_YAML_KEY, BODY_SHELL_TYPE_YAML_KEY, RACER_NAME_YAML_KEY ] for params in force_list_params: yaml_values[params] = force_list(yaml_values[params]) # Populate the model_metadata_s3_key values to handle both training and evaluation for all race_formats if None in yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY]: # MODEL_METADATA_FILE_S3_KEY not passed as part of yaml file ==> This happens during evaluation # Assume model_metadata.json is present in the s3_prefix/model/ folder yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY] = list() for s3_prefix in yaml_values[MODEL_S3_PREFIX_YAML_KEY]: yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY].append( os.path.join(s3_prefix, 'model/model_metadata.json')) # Set multicar value if there is more than one value in MODEL_S3_BUCKET_YAML_KEY. multicar = len(yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 # Set f1 as true if RACE_TYPE is F1 is_f1 = yaml_values[RACE_TYPE_YAML_KEY] == F1_RACE_TYPE # Validate the yaml values validate_yaml_values(yaml_values, multicar) # List of racecar names that should include second camera while launching racecars_with_stereo_cameras = list() # List of racecar names that should include lidar while launching racecars_with_lidars = list() # List of SimApp versions simapp_versions = list() # List of body shell types body_shell_types = yaml_values[BODY_SHELL_TYPE_YAML_KEY] racer_names = yaml_values[RACER_NAME_YAML_KEY] if None in body_shell_types: # use default shells if None in racer_names: body_shell_types = [BodyShellType.DEFAULT.value] * len( yaml_values[MODEL_S3_BUCKET_YAML_KEY]) # use default shells for regular user and f1 shell for users in F1_SHELL_USERS_LIST else: body_shell_types = [ BodyShellType.F1_2021.value if racer_alias in F1_SHELL_USERS_LIST else BodyShellType.DEFAULT.value for racer_alias in yaml_values[RACER_NAME_YAML_KEY] ] yaml_dict[BODY_SHELL_TYPE_YAML_KEY] = body_shell_types # override local yaml file with updated BODY_SHELL_TYPE with open(local_yaml_path, 'w') as yaml_file: yaml.dump(yaml_dict, yaml_file) for agent_index, model_s3_bucket in enumerate( yaml_values[MODEL_S3_BUCKET_YAML_KEY]): racecar_name = 'racecar_' + str(agent_index) if len( yaml_values[MODEL_S3_BUCKET_YAML_KEY]) > 1 else 'racecar' # Make a local folder with the racecar name to download the model_metadata.json if not os.path.exists(os.path.join(os.getcwd(), racecar_name)): os.makedirs(os.path.join(os.getcwd(), racecar_name)) local_model_metadata_path = os.path.abspath( os.path.join(os.path.join(os.getcwd(), racecar_name), 'model_metadata.json')) json_key = yaml_values[MODEL_METADATA_FILE_S3_YAML_KEY][ agent_index] json_key = json_key.replace('s3://{}/'.format(model_s3_bucket), '') s3_client.download_file(Bucket=model_s3_bucket, Key=json_key, Filename=local_model_metadata_path) sensors, _, simapp_version = utils_parse_model_metadata.parse_model_metadata( local_model_metadata_path) simapp_versions.append(simapp_version) if Input.STEREO.value in sensors: racecars_with_stereo_cameras.append(racecar_name) if Input.LIDAR.value in sensors or Input.SECTOR_LIDAR.value in sensors: racecars_with_lidars.append(racecar_name) cmd = [ ''.join(("roslaunch deepracer_simulation_environment {} ".format( launch_name), "local_yaml_path:={} ".format(local_yaml_path), "racecars_with_stereo_cameras:={} ".format( ','.join(racecars_with_stereo_cameras)), "racecars_with_lidars:={} ".format( ','.join(racecars_with_lidars)), "multicar:={} ".format(multicar), "body_shell_types:={} ".format( ','.join(body_shell_types)), "simapp_versions:={} ".format(','.join(simapp_versions)), "f1:={}".format(is_f1))) ] Popen(cmd, shell=True, executable="/bin/bash") except botocore.exceptions.ClientError as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400) except botocore.exceptions.EndpointConnectionError: log_and_exit("No Internet connection or s3 service unavailable", SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500) except Exception as ex: log_and_exit( "Download params and launch of agent node failed: s3_bucket: {}, yaml_key: {}, {}" .format(s3_bucket, yaml_key, ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500)