def write_metrics_to_s3(bucket, key, region, metrics):
'''Helper method that uploads the desired metrics to s3
bucket - String with S3 bucket where metrics should be written
key - String with S3 bucket key where metrics should be written
region - String with aws region
metrics - Dictionary with metrics to write to s3
'''
try:
session = boto3.session.Session()
s3_client = session.client('s3',
region_name=region,
config=get_boto_config())
s3_client.put_object(Bucket=bucket,
Key=key,
Body=bytes(json.dumps(metrics), encoding='utf-8'))
except botocore.exceptions.ClientError as err:
log_and_exit(
"Unable to write metrics to s3: bucket: \
{}, error: {}".format(bucket,
err.response['Error']['Code']),
SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400)
except Exception as ex:
log_and_exit("Unable to write metrics to s3, exception: {}".format(ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def __init__(self,
params: S3BotoDataStoreParameters,
graph_manager: MultiAgentGraphManager,
ignore_lock: bool = False):
self.params = params
self.key_prefixes = dict()
self.ip_data_keys = dict()
self.ip_done_keys = dict()
self.preset_data_keys = dict()
self.delete_queues = dict()
for agent_key, s3_folder in self.params.s3_folders.items():
self.key_prefixes[agent_key] = os.path.join(s3_folder, "model")
self.ip_data_keys[agent_key] = os.path.join(
s3_folder, "ip/ip.json")
self.ip_done_keys[agent_key] = os.path.join(s3_folder, "ip/done")
self.preset_data_keys[agent_key] = os.path.join(
s3_folder, "presets/preset.py")
self.delete_queues[agent_key] = queue.Queue()
if not graph_manager:
log_and_exit("None type for graph manager",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
self.graph_manager = graph_manager
self.ignore_lock = ignore_lock
def _take_action(self, action_list):
try:
# Update state, reward, done flag
self.state = list()
self.reward = list()
self.done = list()
[agent.step(None) for agent in self.non_trainable_agents]
for agent, action in zip(self.agent_list, action_list):
next_state, reward, done = agent.step(action)
self.state.append(next_state)
self.reward.append(reward)
self.done.append(done)
# Preserve behavior of TimeLimit wrapper
self.steps += 1
if MAX_STEPS <= self.steps:
self.done = [True] * self.num_agents
# Terminate the episode if any agent is done
if any(self.done):
[agent.finish_episode() for agent in self.non_trainable_agents]
[agent.finish_episode() for agent in self.agent_list]
except GenericTrainerException as ex:
ex.log_except_and_exit()
except GenericRolloutException as ex:
ex.log_except_and_exit()
except RewardFunctionError as err:
err.log_except_and_exit()
except Exception as ex:
log_and_exit('Unclassified exception: {}'.format(ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def get_chkpoint_num(self, agent_key):
try:
s3_client = self._get_client()
# If there is a lock file return -1 since it means the trainer has the lock
response = s3_client.list_objects_v2(Bucket=self.params.buckets[agent_key],
Prefix=self._get_s3_key(SyncFiles.LOCKFILE.value, agent_key))
chkpoint_num = -1
if "Contents" not in response:
base_checkpoint_dir = self.params.base_checkpoint_dir
checkpoint_dir = base_checkpoint_dir if len(self.graph_manager.agents_params) == 1 else os.path.join(base_checkpoint_dir, agent_key)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
state_file = CheckpointStateFile(os.path.abspath(checkpoint_dir))
s3_client.download_file(Bucket=self.params.buckets[agent_key],
Key=self._get_s3_key(state_file.filename, agent_key),
Filename=state_file.path)
checkpoint_state = state_file.read()
if checkpoint_state is not None:
chkpoint_num = checkpoint_state.num
return chkpoint_num
except botocore.exceptions.ClientError:
log_and_exit("Unable to download checkpoint",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
except Exception:
log_and_exit("Unable to download checkpoint",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def main():
try:
# parse argument
race_idx = sys.argv[1]
race_yaml_path = sys.argv[2]
racecars_with_stereo_cameras = sys.argv[3]
racecars_with_lidars = sys.argv[4]
race_car_colors = sys.argv[5]
simapp_versions = sys.argv[6]
launch_name = 'tournament_rl_agent.launch'
cmd = ' '.join([
"roslaunch", "deepracer_simulation_environment",
"{}".format(launch_name),
"local_yaml_path:={}".format(race_yaml_path),
"racecars_with_stereo_cameras:={}".format(
racecars_with_stereo_cameras),
"racecars_with_lidars:={}".format(racecars_with_lidars),
"multicar:={}".format(True),
"car_colors:={}".format(race_car_colors),
"simapp_versions:={}".format(simapp_versions)
])
logger.info("cmd: {}".format(cmd))
subprocess.Popen(cmd, shell=True, executable="/bin/bash")
except Exception as e:
log_and_exit(
"Tournament race node failed: race_idx: {}, {}".format(
race_idx, e), SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def get_yaml_dict(local_yaml_path):
'''local_yaml_path - path to the local yaml file
'''
with open(local_yaml_path, 'r') as stream:
try:
return yaml.safe_load(stream)
except yaml.YAMLError as exc:
log_and_exit("yaml read error: {}".format(exc),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def download_customer_reward_function(s3_client, reward_file_s3_key):
reward_function_local_path_preprocessed = os.path.join(
CUSTOM_FILES_PATH, "customer_reward_function_preprocessed.py")
success_reward_function_download = s3_client.download_file(
s3_key=reward_file_s3_key,
local_path=reward_function_local_path_preprocessed)
if not success_reward_function_download:
utils.log_and_exit("Unable to download the reward function code.",
utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_400)
fstring_decoded_reward_function(reward_function_local_path_preprocessed)
def upload_hyperparameters(self, hyperparams_json):
try:
s3_client = self.get_client()
file_handle = io.BytesIO(hyperparams_json.encode())
s3_client.upload_fileobj(file_handle, self.bucket,
self.hyperparameters_key)
except botocore.exceptions.ClientError:
log_and_exit("Hyperparameters failed to upload",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
except Exception:
log_and_exit("Hyperparameters failed to upload",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def get_yaml_values(yaml_dict, default_vals=None):
'''yaml_dict - dict containing yaml configs
default_vals - Dictionary of the default values to be used if key is not present
'''
try:
return_values = dict()
default_val_keys = default_vals.keys() if default_vals else []
for key in default_val_keys:
return_values[key] = yaml_dict.get(key, default_vals[key])
return return_values
except yaml.YAMLError as exc:
log_and_exit("yaml read error: {}".format(exc),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def __init__(self,
level: LevelSelection,
seed: int,
frame_skip: int,
custom_reward_threshold: Union[int, float],
visualization_parameters: VisualizationParameters,
agents_params: List[AgentParameters],
non_trainable_agents: List[Agent],
run_phase_subject: RunPhaseSubject,
target_success_rate: float = 1.0,
**kwargs):
super().__init__(level,
seed,
frame_skip,
custom_reward_threshold,
visualization_parameters,
target_success_rate,
num_agents=len(agents_params))
try:
# Maintain a list of all agents
self.agent_list = [
agent_param.env_agent for agent_param in agents_params
]
self.action_list = [None for agent_param in agents_params]
self.non_trainable_agents = non_trainable_agents
# Create the state and action space
self.state_space = [
agent.get_observation_space() for agent in self.agent_list
]
self.action_space = [
agent.get_action_space() for agent in self.agent_list
]
# Agents' info map
self._agents_info_map = {}
# Set the phase subject to notify observers when phase changes
self.run_phase_subject = run_phase_subject
# Initialize step count
self.steps = 0
# Initialize the state by getting a new state from the environment
self.reset_internal_state(True)
except GenericTrainerException as ex:
ex.log_except_and_exit()
except GenericRolloutException as ex:
ex.log_except_and_exit()
except Exception as ex:
log_and_exit('Unclassified exception: {}'.format(ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def fstring_decoded_reward_function(reward_function_local_path_preprocessed):
try:
reward_function_local_path_processed = os.path.join(
CUSTOM_FILES_PATH, "customer_reward_function.py")
with open(reward_function_local_path_preprocessed,
'rb') as filepointer:
text, _ = future_fstrings.fstring_decode(filepointer.read())
with open(reward_function_local_path_processed, 'wb') as filepointer:
filepointer.write(text.encode('UTF-8'))
except Exception as e:
utils.log_and_exit(
"Failed to decode the fstring format in reward function: {}".
format(e), utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500)
def download_customer_reward_function(s3_client, reward_file_s3_key):
""" Download the customer reward function from s3 bucket
Arguments:
s3_client {[s3_session]} -- [S3 session object]
reward_file_s3_key {[str]} -- [Reward function file path in s3]
"""
reward_function_local_path_preprocessed = os.path.join(CUSTOM_FILES_PATH,
"customer_reward_function_preprocessed.py")
success_reward_function_download = s3_client.download_file(s3_key=reward_file_s3_key,
local_path=reward_function_local_path_preprocessed)
if not success_reward_function_download:
utils.log_and_exit("Unable to download the reward function code.",
utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_400)
fstring_decoded_reward_function(reward_function_local_path_preprocessed)
def __call__(self, *argv):
''' Makes a client call for the stored service
argv - Arguments to pass into the client object
'''
try:
return self.client(*argv)
except TypeError as err:
log_and_exit("Invalid arguments for client {}".format(err),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
except Exception as ex:
time.sleep(ROBOMAKER_CANCEL_JOB_WAIT_TIME)
log_and_exit("Unable to call service {}".format(ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def upload_file(self, s3_key, local_path):
s3_client = self.get_client()
try:
s3_client.upload_file(Filename=local_path,
Bucket=self.bucket,
Key=s3_key)
return True
except botocore.exceptions.ClientError:
log_and_exit("Unable to upload file",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
except Exception:
log_and_exit("Unable to upload file",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def upload_finished_file(self):
try:
s3_client = self._get_client()
for agent_key, bucket in self.params.buckets.items():
s3_client.upload_fileobj(Fileobj=io.BytesIO(b''),
Bucket=bucket,
Key=self._get_s3_key(SyncFiles.FINISHED.value, agent_key))
except botocore.exceptions.ClientError:
log_and_exit("Unable to upload finish file",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
except Exception:
log_and_exit("Unable to upload finish file",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def main(racecar_names):
""" Main function for kinesis_video_camera
Arguments:
racecar_names (str): racecar_names as a comma seperated string
"""
try:
racecars_info = get_racecars_info(racecar_names)
for racecar in racecars_info:
# Instantiate KinesisVideoCamera objects for each racecar
KinesisVideoCamera(racecar['name'], racecars_info)
except Exception as err_msg:
log_and_exit(
"Exception in Kinesis Video camera ros node: {}".format(err_msg),
SIMAPP_SIMULATION_KINESIS_VIDEO_CAMERA_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def write_simtrace_data(self, jsondata):
if self.data_state != SIMTRACE_DATA_UPLOAD_UNKNOWN_STATE:
try:
csvdata = []
for key in SIMTRACE_CSV_DATA_HEADER:
csvdata.append(jsondata[key])
self.csvwriter.writerow(csvdata)
self.total_upload_size += sys.getsizeof(csvdata)
logger.debug("csvdata={} size data={} csv={}".format(
csvdata, sys.getsizeof(csvdata),
sys.getsizeof(self.simtrace_csv_data.getvalue())))
except Exception:
utils.log_and_exit(
"Invalid SIM_TRACE data format",
utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500)
def _restart_environment_episode(self, force_environment_reset=False):
try:
self.steps = 0
[agent.reset_agent() for agent in self.non_trainable_agents]
self.state = [agent.reset_agent() for agent in self.agent_list]
# Reset state, reward, done flag
self.reward = [0.0] * self.num_agents
self.done = [False] * self.num_agents
except GenericTrainerException as ex:
ex.log_except_and_exit()
except GenericRolloutException as ex:
ex.log_except_and_exit()
except Exception as ex:
log_and_exit('Unclassified exception: {}'.format(ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def upload_to_s3(self):
''' This will upload all the files provided parallely using multiprocess
'''
# Continue uploading other files if one of the file does not exists
try:
multiprocess_pool = Pool(MULTIPROCESS_S3WRITER_POOL)
multiprocess_pool.starmap(self._multiprocess_upload_s3,
[(job.s3_bucket, job.s3_prefix, job.aws_region, job.local_file)
for job in self.job_info])
multiprocess_pool.close()
multiprocess_pool.join()
_ = [os.remove(job.local_file) for job in self.job_info]
self.upload_num += 1
except Exception as ex:
log_and_exit('Unclassified exception: {}'.format(ex), SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def _take_action(self, action_list):
try:
self.action_list = action_list
# action: send action to gazebo
[agent.send_action(None) for agent in self.non_trainable_agents]
[
agent.send_action(action)
for agent, action in zip(self.agent_list, self.action_list)
]
except GenericTrainerException as ex:
ex.log_except_and_exit()
except GenericRolloutException as ex:
ex.log_except_and_exit()
except Exception as ex:
log_and_exit('Unclassified exception: {}'.format(ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def _update_state(self):
try:
self.state = list()
self.reward = list()
self.done = list()
# trainable agent physics: update agent status
[
self._agents_info_map.update(agent.update_agent(action))
for agent, action in zip(self.agent_list, self.action_list)
]
# non-trainable agent physics: update agent status
[
self._agents_info_map.update(agent.update_agent(None))
for agent in self.non_trainable_agents
]
# trainable agent judge: comparision between action and physics for reset
for agent, action in zip(self.agent_list, self.action_list):
next_state, reward, done = agent.judge_action(
action, self._agents_info_map)
self.state.append(next_state)
self.reward.append(reward)
self.done.append(done)
# non-trainable agent judge: for bot car and obstacles
[
agent.judge_action(action, self._agents_info_map)
for agent in self.non_trainable_agents
]
# Preserve behavior of TimeLimit wrapper
self.steps += 1
if MAX_STEPS <= self.steps:
self.done = [True] * self.num_agents
# Terminate the episode if any agent is done
if any(self.done):
[agent.finish_episode() for agent in self.non_trainable_agents]
[agent.finish_episode() for agent in self.agent_list]
except GenericTrainerException as ex:
ex.log_except_and_exit()
except GenericRolloutException as ex:
ex.log_except_and_exit()
except RewardFunctionError as err:
err.log_except_and_exit()
except Exception as ex:
log_and_exit('Unclassified exception: {}'.format(ex),
SIMAPP_SIMULATION_WORKER_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def write_ip_config(self, ip_address):
try:
s3_client = self.get_client()
data = {"IP": ip_address}
json_blob = json.dumps(data)
file_handle = io.BytesIO(json_blob.encode())
file_handle_done = io.BytesIO(b'done')
s3_client.upload_fileobj(file_handle, self.bucket, self.config_key)
s3_client.upload_fileobj(file_handle_done, self.bucket,
self.done_file_key)
except botocore.exceptions.ClientError:
log_and_exit("Write ip config failed to upload",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
except Exception:
log_and_exit("Write ip config failed to upload",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def download_file(self, s3_key, local_path):
s3_client = self.get_client()
try:
s3_client.download_file(self.bucket, s3_key, local_path)
return True
except botocore.exceptions.ClientError as err:
# It is possible that the file isn't there in which case we should
# return fasle and let the client decide the next action
if err.response['Error']['Code'] == "404":
return False
else:
log_and_exit("Unable to download file",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
except Exception:
log_and_exit("Unable to download file",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def fstring_decoded_reward_function(reward_function_local_path_preprocessed):
""" python 3.6 supports fstring and console lambda function validates using python3.6.
But all the simapp code is runs in python3.5 which does not support fstring. This funciton
support fstring in python 3.5
Arguments:
reward_function_local_path_preprocessed {[str]} -- [Reward function file path]
"""
try:
reward_function_local_path_processed = os.path.join(CUSTOM_FILES_PATH, "customer_reward_function.py")
with open(reward_function_local_path_preprocessed, 'rb') as filepointer:
text, _ = future_fstrings.fstring_decode(filepointer.read())
with open(reward_function_local_path_processed, 'wb') as filepointer:
filepointer.write(text.encode('UTF-8'))
except Exception as e:
utils.log_and_exit("Failed to decode the fstring format in reward function: {}".format(e),
utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500)
def main(racecar_names):
""" Main function """
try:
for racecar_name in racecar_names.split(','):
agent_name = 'agent' if len(racecar_name.split("_")) == 1 else "agent_{}".format(racecar_name.split("_")[1])
camera_info = utils.get_cameratype_params(racecar_name, agent_name)
save_to_mp4_obj = SaveToMp4(camera_infos=[camera_info[CameraTypeParams.CAMERA_PIP_PARAMS],
camera_info[CameraTypeParams.CAMERA_45DEGREE_PARAMS],
camera_info[CameraTypeParams.CAMERA_TOPVIEW_PARAMS]],
fourcc=Mp4Parameter.FOURCC.value,
fps=Mp4Parameter.FPS.value,
frame_size=Mp4Parameter.FRAME_SIZE.value)
rospy.Service('/{}/save_mp4/subscribe_to_save_mp4'.format(racecar_name),
Empty, save_to_mp4_obj.subscribe_to_save_mp4)
rospy.Service('/{}/save_mp4/unsubscribe_from_save_mp4'.format(racecar_name),
Empty, save_to_mp4_obj.unsubscribe_to_save_mp4)
except Exception as err_msg:
log_and_exit("Exception in save_mp4 ros node: {}".format(err_msg),
SIMAPP_SIMULATION_SAVE_TO_MP4_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def subscribe_to_save_mp4(self, req):
""" Ros service handler function used to subscribe to the Image topic.
Arguments:
req (req): Dummy req else the ros service throws exception
Return:
[] - Empty list else ros service throws exception
"""
try:
for camera_enum in self.camera_infos:
name = camera_enum['name']
local_path, topic_name = camera_enum['local_path'], camera_enum['topic_name']
self.cv2_video_writers[name] = cv2.VideoWriter(local_path, self.fourcc,
self.fps, self.frame_size)
self.mp4_subscription[name] = rospy.Subscriber(topic_name, Image,
callback=self._subscribe_to_image_topic,
callback_args=name)
return []
except Exception as err_msg:
log_and_exit("Exception in the handler function to subscribe to save_mp4 download: {}".format(err_msg),
SIMAPP_SIMULATION_SAVE_TO_MP4_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def get_ip(self):
s3_client = self.get_client()
time_elapsed = 0
try:
# Wait for sagemaker to produce the redis ip
while time_elapsed < SAGEMAKER_WAIT_TIME:
response = s3_client.list_objects(Bucket=self.bucket,
Prefix=self.done_file_key)
if "Contents" in response:
break
time.sleep(1)
time_elapsed += 1
if time_elapsed % 5 == 0:
LOG.info(
"Waiting for SageMaker Redis server IP: Time elapsed: %s seconds",
time_elapsed)
if time_elapsed >= SAGEMAKER_WAIT_TIME:
log_and_exit(
"Timed out while attempting to retrieve the Redis IP",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
# Download the ip file
s3_client.download_file(self.bucket, self.config_key, 'ip.json')
with open("ip.json") as file:
ip_file = json.load(file)["IP"]
return ip_file
except botocore.exceptions.ClientError:
log_and_exit("Unable to retrieve redis ip",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_400)
except Exception:
log_and_exit("Unable to retrieve redis ip",
SIMAPP_S3_DATA_STORE_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def unsubscribe_to_save_mp4(self, req):
""" Ros service handler function used to unsubscribe from the Image topic.
This will take care of cleaning and releasing the cv2 VideoWriter
Arguments:
req (req): Dummy req else the ros service throws exception
Return:
[] - Empty list else ros service throws exception
"""
try:
for camera_enum in self.camera_infos:
name = camera_enum['name']
if name in self.mp4_subscription:
self.mp4_subscription[name].unregister()
del self.mp4_subscription[name]
if name in self.cv2_video_writers:
self.cv2_video_writers[name].release()
del self.cv2_video_writers[name]
return []
except Exception as err_msg:
log_and_exit("Exception in the handler function to unsubscribe from save_mp4 download: {}".format(err_msg),
SIMAPP_SIMULATION_SAVE_TO_MP4_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
def wait_for_checkpoints(checkpoint_dirs, data_store=None, timeout=10):
"""
block until there is a checkpoint in all of the checkpoint_dirs
"""
chkpt_state_files = [CheckpointStateFile(checkpoint_dir) for checkpoint_dir in checkpoint_dirs]
for i in range(timeout):
if data_store:
data_store.load_from_store()
all_agent_checkpoint_copied = all([chkpt_state_file.read() is not None for chkpt_state_file in chkpt_state_files])
if all_agent_checkpoint_copied:
return
time.sleep(10)
# one last time
all_agent_checkpoint_copied = all([chkpt_state_file.read() is not None for chkpt_state_file in chkpt_state_files])
if all_agent_checkpoint_copied:
return
utils.log_and_exit("Checkpoint never found in {} : {}, waited {} seconds." \
.format(checkpoint_dirs, all_agent_checkpoint_copied, timeout),
utils.SIMAPP_SIMULATION_WORKER_EXCEPTION,
utils.SIMAPP_EVENT_ERROR_CODE_500)
def get_image(icon_name, img_size=None):
""" Given the icon_name in the track_iconography folder without png, gives back cv2 image
with all 4 channels
Args:
icon_name (str): The name of the icon in the track_iconography folder
img_size (tuple): If you want to resize the image (width, height) (default: {None})
Returns:
Image: The cv2 image read from the .png file
"""
try:
track_iconography_dir = os.path.join(
rospkg.RosPack().get_path('deepracer_simulation_environment'),
'track_iconography')
image_path = os.path.join(track_iconography_dir, icon_name + '.png')
image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
if img_size:
image = cv2.resize(image, img_size)
return image
except (OSError, IOError, Exception) as err_msg:
log_and_exit(
"Iconography image does not exists or corrupt image: {}".format(
err_msg), SIMAPP_SIMULATION_SAVE_TO_MP4_EXCEPTION,
SIMAPP_EVENT_ERROR_CODE_500)
