def __init__(self,
agent_name,
s3_dict_metrics,
is_continuous,
pause_time_before_start=0.0):
"""Init eval metrics
Args:
agent_name (string): agent name
s3_dict_metrics (dict): Dictionary containing the required
s3 info for the metrics bucket with keys specified by MetricsS3Keys
is_continuous (bool): True if continuous race, False otherwise
pause_time_before_start (float): second to pause before race start
"""
self._pause_time_before_start = pause_time_before_start
self._is_pause_time_subtracted = False
self._agent_name_ = agent_name
self._s3_metrics = Metrics(
bucket=s3_dict_metrics[MetricsS3Keys.METRICS_BUCKET.value],
s3_key=s3_dict_metrics[MetricsS3Keys.METRICS_KEY.value],
region_name=s3_dict_metrics[MetricsS3Keys.REGION.value],
)
self._is_continuous = is_continuous
self._start_time_ = time.time()
self._number_of_trials_ = 0
self._progress_ = 0.0
self._episode_status = ""
self._metrics_ = list()
# This is used to calculate the actual distance traveled by the car
self._agent_xy = list()
self._prev_step_time = time.time()
self.is_save_simtrace_enabled = rospy.get_param(
"SIMTRACE_S3_BUCKET", None)
# Create the agent specific directories needed for storing the metric files
self._simtrace_local_path = SIMTRACE_EVAL_LOCAL_PATH_FORMAT.format(
self._agent_name_)
simtrace_dirname = os.path.dirname(self._simtrace_local_path)
if simtrace_dirname or not os.path.exists(simtrace_dirname):
os.makedirs(simtrace_dirname)
self.reset_count_dict = {
EpisodeStatus.CRASHED.value: 0,
EpisodeStatus.OFF_TRACK.value: 0,
EpisodeStatus.IMMOBILIZED.value: 0,
EpisodeStatus.REVERSED.value: 0,
}
self._best_lap_time = float("inf")
self._total_evaluation_time = 0
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
self._reset_count_sum = 0
self._current_sim_time = 0
self.track_data = TrackData.get_instance()
rospy.Service(
"/{}/{}".format(self._agent_name_, "mp4_video_metrics"),
VideoMetricsSrv,
self._handle_get_video_metrics,
)
AbstractTracker.__init__(self, TrackerPriority.HIGH)
def __init__(self,
agent_name,
s3_dict_metrics,
deepracer_checkpoint_json,
ckpnt_dir,
run_phase_sink,
use_model_picker=True):
'''s3_dict_metrics - Dictionary containing the required s3 info for the metrics
bucket with keys specified by MetricsS3Keys
deepracer_checkpoint_json - DeepracerCheckpointJson instance
ckpnt_dir - Directory where the current checkpont is to be stored
run_phase_sink - Sink to recieve notification of a change in run phase
use_model_picker - Flag to whether to use model picker or not.
'''
self._agent_name_ = agent_name
self._deepracer_checkpoint_json = deepracer_checkpoint_json
self._s3_metrics = Metrics(
bucket=s3_dict_metrics[MetricsS3Keys.METRICS_BUCKET.value],
s3_key=s3_dict_metrics[MetricsS3Keys.METRICS_KEY.value],
region_name=s3_dict_metrics[MetricsS3Keys.REGION.value])
self._start_time_ = time.time()
self._episode_ = 0
self._episode_reward_ = 0.0
self._progress_ = 0.0
self._episode_status = ''
self._metrics_ = list()
self._is_eval_ = True
self._eval_trials_ = 0
self._checkpoint_state_ = CheckpointStateFile(ckpnt_dir)
self._use_model_picker = use_model_picker
self._eval_stats_dict_ = {'chkpnt_name': None, 'avg_eval_metric': None}
self._best_chkpnt_stats = {
'name': None,
'avg_eval_metric': None,
'time_stamp': time.time()
}
self._current_eval_best_model_metric_list_ = list()
self.is_save_simtrace_enabled = rospy.get_param(
'SIMTRACE_S3_BUCKET', None)
self._best_model_metric_type = BestModelMetricType(
rospy.get_param('BEST_MODEL_METRIC',
BestModelMetricType.PROGRESS.value).lower())
self.track_data = TrackData.get_instance()
run_phase_sink.register(self)
# Create the agent specific directories needed for storing the metric files
self._simtrace_local_path = SIMTRACE_TRAINING_LOCAL_PATH_FORMAT.format(
self._agent_name_)
simtrace_dirname = os.path.dirname(self._simtrace_local_path)
if simtrace_dirname or not os.path.exists(simtrace_dirname):
os.makedirs(simtrace_dirname)
self._current_sim_time = 0
rospy.Service("/{}/{}".format(self._agent_name_, "mp4_video_metrics"),
VideoMetricsSrv, self._handle_get_video_metrics)
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
AbstractTracker.__init__(self, TrackerPriority.HIGH)
def __init__(self, agent_name, s3_dict_metrics, is_continuous):
'''Init eval metrics
Args:
agent_name (string): agent name
s3_dict_metrics (dict): Dictionary containing the required
s3 info for the metrics bucket with keys specified by MetricsS3Keys
is_continuous (bool): True if continuous race, False otherwise
'''
self._agent_name_ = agent_name
self._s3_dict_metrics_ = s3_dict_metrics
self._is_continuous = is_continuous
self._start_time_ = time.time()
self._number_of_trials_ = 0
self._progress_ = 0.0
self._episode_status = ''
self._metrics_ = list()
# This is used to calculate the actual distance traveled by the car
self._agent_xy = list()
self._prev_step_time = None
self.is_save_simtrace_enabled = rospy.get_param(
'SIMTRACE_S3_BUCKET', None)
# Create the agent specific directories needed for storing the metric files
simtrace_dirname = os.path.dirname(
IterationDataLocalFileNames.SIM_TRACE_EVALUATION_LOCAL_FILE.value)
if not os.path.exists(
os.path.join(ITERATION_DATA_LOCAL_FILE_PATH, self._agent_name_,
simtrace_dirname)):
os.makedirs(
os.path.join(ITERATION_DATA_LOCAL_FILE_PATH, self._agent_name_,
simtrace_dirname))
self.reset_count_dict = {
EpisodeStatus.CRASHED.value: 0,
EpisodeStatus.OFF_TRACK.value: 0,
EpisodeStatus.IMMOBILIZED.value: 0,
EpisodeStatus.REVERSED.value: 0
}
self._best_lap_time = float('inf')
self._total_evaluation_time = 0
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
self._reset_count_sum = 0
rospy.Service("/{}/{}".format(self._agent_name_, "mp4_video_metrics"),
VideoMetricsSrv, self._handle_get_video_metrics)
def clear(self):
"""clear all EvalMetrics member variable"""
self._is_pause_time_subtracted = False
self._start_time_ = self._current_sim_time
self._number_of_trials_ = 0
self._progress_ = 0.0
self._episode_status = ""
self._metrics_ = list()
self._agent_xy = list()
self._prev_step_time = self._current_sim_time
self.reset_count_dict = {
EpisodeStatus.CRASHED.value: 0,
EpisodeStatus.OFF_TRACK.value: 0,
EpisodeStatus.IMMOBILIZED.value: 0,
EpisodeStatus.REVERSED.value: 0,
}
self._best_lap_time = float("inf")
self._total_evaluation_time = 0
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
self._reset_count_sum = 0
self._current_sim_time = 0
def __init__(self, agent_name, s3_dict_metrics):
'''s3_dict_metrics - Dictionary containing the required s3 info for the metrics
bucket with keys specified by MetricsS3Keys
'''
self._agent_name_ = agent_name
self._s3_dict_metrics_ = s3_dict_metrics
self._start_time_ = time.time()
self._number_of_trials_ = 0
self._progress_ = 0.0
self._episode_status = ''
self._metrics_ = list()
# This is used to calculate the actual distance travelled by the car
self._agent_xy = list()
self._prev_step_time = None
self._simtrace_data_ = \
DeepRacerRacetrackSimTraceData(self._s3_dict_metrics_[MetricsS3Keys.STEP_BUCKET.value],
self._s3_dict_metrics_[MetricsS3Keys.STEP_KEY.value],
self._s3_dict_metrics_[MetricsS3Keys.ENDPOINT_URL.value])
# Create the agent specific directories needed for storing the metric files
simtrace_dirname = os.path.dirname(
IterationDataLocalFileNames.SIM_TRACE_EVALUATION_LOCAL_FILE.value)
if not os.path.exists(
os.path.join(ITERATION_DATA_LOCAL_FILE_PATH, self._agent_name_,
simtrace_dirname)):
os.makedirs(
os.path.join(ITERATION_DATA_LOCAL_FILE_PATH, self._agent_name_,
simtrace_dirname))
self.reset_count_dict = {
EpisodeStatus.CRASHED.value: 0,
EpisodeStatus.OFF_TRACK.value: 0,
EpisodeStatus.IMMOBILIZED.value: 0,
EpisodeStatus.REVERSED.value: 0
}
self._best_lap_time = float('inf')
self._total_evaluation_time = 0
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
rospy.Service("/{}/{}".format(self._agent_name_, "mp4_video_metrics"),
VideoMetricsSrv, self._handle_get_video_metrics)
def __init__(self, agent_name, s3_dict_metrics, s3_dict_model, ckpnt_dir, run_phase_sink, use_model_picker=True):
'''s3_dict_metrics - Dictionary containing the required s3 info for the metrics
bucket with keys specified by MetricsS3Keys
s3_dict_model - Dictionary containing the required s3 info for the model
bucket, which is where the best model info will be saved with
keys specified by MetricsS3Keys
ckpnt_dir - Directory where the current checkpont is to be stored
run_phase_sink - Sink to recieve notification of a change in run phase
use_model_picker - Flag to whether to use model picker or not.
'''
self._agent_name_ = agent_name
self._s3_dict_metrics_ = s3_dict_metrics
self._s3_dict_model_ = s3_dict_model
self._start_time_ = time.time()
self._episode_ = 0
self._episode_reward_ = 0.0
self._progress_ = 0.0
self._episode_status = ''
self._metrics_ = list()
self._is_eval_ = True
self._eval_trials_ = 0
self._checkpoint_state_ = CheckpointStateFile(ckpnt_dir)
self._use_model_picker = use_model_picker
self._eval_stats_dict_ = {'chkpnt_name': None, 'avg_comp_pct': -1.0}
self._best_chkpnt_stats = {'name': None, 'avg_comp_pct': -1.0, 'time_stamp': time.time()}
self._current_eval_pct_list_ = list()
self.is_save_simtrace_enabled = rospy.get_param('SIMTRACE_S3_BUCKET', None)
self.track_data = TrackData.get_instance()
run_phase_sink.register(self)
# Create the agent specific directories needed for storing the metric files
simtrace_dirname = os.path.dirname(IterationDataLocalFileNames.SIM_TRACE_TRAINING_LOCAL_FILE.value)
if not os.path.exists(os.path.join(ITERATION_DATA_LOCAL_FILE_PATH, self._agent_name_, simtrace_dirname)):
os.makedirs(os.path.join(ITERATION_DATA_LOCAL_FILE_PATH, self._agent_name_, simtrace_dirname))
self._current_sim_time = 0
rospy.Service("/{}/{}".format(self._agent_name_, "mp4_video_metrics"), VideoMetricsSrv,
self._handle_get_video_metrics)
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
AbstractTracker.__init__(self, TrackerPriority.HIGH)
def __init__(self,
agent_name,
s3_dict_metrics,
deepracer_checkpoint_json,
ckpnt_dir,
run_phase_sink,
use_model_picker=True):
'''s3_dict_metrics - Dictionary containing the required s3 info for the metrics
bucket with keys specified by MetricsS3Keys
deepracer_checkpoint_json - DeepracerCheckpointJson instance
ckpnt_dir - Directory where the current checkpont is to be stored
run_phase_sink - Sink to recieve notification of a change in run phase
use_model_picker - Flag to whether to use model picker or not.
'''
self._agent_name_ = agent_name
self._deepracer_checkpoint_json = deepracer_checkpoint_json
self._s3_metrics = Metrics(
bucket=s3_dict_metrics[MetricsS3Keys.METRICS_BUCKET.value],
s3_key=s3_dict_metrics[MetricsS3Keys.METRICS_KEY.value],
region_name=s3_dict_metrics[MetricsS3Keys.REGION.value],
s3_endpoint_url=s3_dict_metrics[MetricsS3Keys.ENDPOINT_URL.value])
self._start_time_ = time.time()
self._episode_ = 0
self._episode_reward_ = 0.0
self._progress_ = 0.0
self._episode_status = ''
self._metrics_ = list()
self._is_eval_ = True
self._eval_trials_ = 0
self._checkpoint_state_ = CheckpointStateFile(ckpnt_dir)
self._use_model_picker = use_model_picker
self._eval_stats_dict_ = {'chkpnt_name': None, 'avg_eval_metric': None}
self._best_chkpnt_stats = {
'name': None,
'avg_eval_metric': None,
'time_stamp': time.time()
}
self._current_eval_best_model_metric_list_ = list()
self.is_save_simtrace_enabled = rospy.get_param(
'SIMTRACE_S3_BUCKET', None)
self._best_model_metric_type = BestModelMetricType(
rospy.get_param('BEST_MODEL_METRIC',
BestModelMetricType.PROGRESS.value).lower())
self.track_data = TrackData.get_instance()
run_phase_sink.register(self)
# Create the agent specific directories needed for storing the metric files
self._simtrace_local_path = SIMTRACE_TRAINING_LOCAL_PATH_FORMAT.format(
self._agent_name_)
simtrace_dirname = os.path.dirname(self._simtrace_local_path)
# addressing mkdir and check directory race condition:
# https://stackoverflow.com/questions/12468022/python-fileexists-error-when-making-directory/30174982#30174982
# TODO: change this to os.makedirs(simtrace_dirname, exist_ok=True) when we migrate off python 2.7
try:
os.makedirs(simtrace_dirname)
except OSError as e:
if e.errno != errno.EEXIST:
raise
LOGGER.error("File already exist %s", simtrace_dirname)
self._current_sim_time = 0
rospy.Service("/{}/{}".format(self._agent_name_, "mp4_video_metrics"),
VideoMetricsSrv, self._handle_get_video_metrics)
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
AbstractTracker.__init__(self, TrackerPriority.HIGH)
def __init__(self,
agent_name,
s3_dict_metrics,
is_continuous,
pause_time_before_start=0.0):
'''Init eval metrics
Args:
agent_name (string): agent name
s3_dict_metrics (dict): Dictionary containing the required
s3 info for the metrics bucket with keys specified by MetricsS3Keys
is_continuous (bool): True if continuous race, False otherwise
pause_time_before_start (float): second to pause before race start
'''
self._pause_time_before_start = pause_time_before_start
self._is_pause_time_subtracted = False
self._agent_name_ = agent_name
self._s3_metrics = Metrics(
bucket=s3_dict_metrics[MetricsS3Keys.METRICS_BUCKET.value],
s3_key=s3_dict_metrics[MetricsS3Keys.METRICS_KEY.value],
region_name=s3_dict_metrics[MetricsS3Keys.REGION.value],
s3_endpoint_url=s3_dict_metrics[MetricsS3Keys.ENDPOINT_URL.value])
self._is_continuous = is_continuous
self._start_time_ = time.time()
self._number_of_trials_ = 0
self._progress_ = 0.0
self._episode_status = ''
self._metrics_ = list()
# This is used to calculate the actual distance traveled by the car
self._agent_xy = list()
self._prev_step_time = time.time()
self.is_save_simtrace_enabled = rospy.get_param(
'SIMTRACE_S3_BUCKET', None)
# Create the agent specific directories needed for storing the metric files
self._simtrace_local_path = SIMTRACE_EVAL_LOCAL_PATH_FORMAT.format(
self._agent_name_)
simtrace_dirname = os.path.dirname(self._simtrace_local_path)
# addressing mkdir and check directory race condition:
# https://stackoverflow.com/questions/12468022/python-fileexists-error-when-making-directory/30174982#30174982
# TODO: change this to os.makedirs(simtrace_dirname, exist_ok=True) when we migrate off python 2.7
try:
os.makedirs(simtrace_dirname)
except OSError as e:
if e.errno != errno.EEXIST:
raise
LOGGER.error("File already exist %s", simtrace_dirname)
self.reset_count_dict = {
EpisodeStatus.CRASHED.value: 0,
EpisodeStatus.OFF_TRACK.value: 0,
EpisodeStatus.IMMOBILIZED.value: 0,
EpisodeStatus.REVERSED.value: 0
}
self._best_lap_time = float('inf')
self._total_evaluation_time = 0
self._video_metrics = Mp4VideoMetrics.get_empty_dict()
self._reset_count_sum = 0
self._current_sim_time = 0
self.track_data = TrackData.get_instance()
rospy.Service("/{}/{}".format(self._agent_name_, "mp4_video_metrics"),
VideoMetricsSrv, self._handle_get_video_metrics)
AbstractTracker.__init__(self, TrackerPriority.HIGH)