def _download_checkpoint(self, version):
"""Setup the Checkpoint object and selete the best checkpoint.
Args:
version (float): SimApp version
Returns:
Checkpoint: Checkpoint class instance
"""
# download checkpoint from s3
checkpoint = Checkpoint(
bucket=self._current_racer.inputModel.s3BucketName,
s3_prefix=self._current_racer.inputModel.s3KeyPrefix,
region_name=self._region,
agent_name=self._agent_name,
checkpoint_dir=self._local_model_directory)
# make coach checkpoint compatible
if version < SIMAPP_VERSION_2 and not checkpoint.rl_coach_checkpoint.is_compatible(
):
checkpoint.rl_coach_checkpoint.make_compatible(
checkpoint.syncfile_ready)
# get best model checkpoint string
model_checkpoint_name = checkpoint.deepracer_checkpoint_json.get_deepracer_best_checkpoint(
)
# Select the best checkpoint model by uploading rl coach .coach_checkpoint file
model_kms = self._current_racer.inputModel.s3KmsKeyArn if hasattr(
self._current_racer.inputModel, 's3KmsKeyArn') else None
checkpoint.rl_coach_checkpoint.update(
model_checkpoint_name=model_checkpoint_name,
s3_kms_extra_args=utils.get_s3_extra_args(model_kms))
return checkpoint
def upload_race_status(self, status_code, error_name=None, error_details=None):
"""Upload race status into s3.
Args:
status_code (str): Status code for race.
error_name (str, optional): The name of the error if is 4xx or 5xx.
Defaults to "".
error_details (str, optional): The detail message of the error
if is 4xx or 5xx.
Defaults to "".
"""
# persist s3 status file
if error_name is not None and error_details is not None:
status = {RaceStatusKeys.STATUS_CODE.value: status_code,
RaceStatusKeys.ERROR_NAME.value: error_name,
RaceStatusKeys.ERROR_DETAILS.value: error_details}
else:
status = {RaceStatusKeys.STATUS_CODE.value: status_code}
status_json = json.dumps(status)
s3_key = os.path.normpath(os.path.join(self._current_racer.outputStatus.s3KeyPrefix,
S3_RACE_STATUS_FILE_NAME))
race_status_kms = self._current_racer.outputStatus.s3KmsKeyArn if \
hasattr(self._current_racer.outputStatus, 's3KmsKeyArn') else None
self._s3_client.upload_fileobj(bucket=self._current_racer.outputStatus.s3BucketName,
s3_key=s3_key,
fileobj=io.BytesIO(status_json.encode()),
s3_kms_extra_args=utils.get_s3_extra_args(race_status_kms))
LOG.info("[virtual event manager] Successfully uploaded race status file to \
s3 bucket {} with s3 key {}.".format(self._current_racer.outputStatus.s3BucketName,
s3_key))
def upload_to_s3(json_log, s3_crash_status_file_name):
if s3_crash_status_file_name is None or json_log is None:
LOG.info("simapp_exit_gracefully - skipping s3 upload.")
return
# this global variable is added to avoid us running into infinte loop
# because s3 client could call log and exit as well.
global is_upload_to_s3_called
if not is_upload_to_s3_called:
is_upload_to_s3_called = True
try:
# I know this dynamic import can be considered as bad code design
# however, it's needed to playaround the circular import issue
# without large scale code change in all places that import log_and_exit
# TODO: refactor this when we migrate entirely to python 3
from markov import utils
from markov.boto.s3.s3_client import S3Client
LOG.info(
"simapp_exit_gracefully - first time upload_to_s3 called.")
s3_client = S3Client()
s3_key = os.path.normpath(
os.path.join(os.environ.get("YAML_S3_PREFIX", ''),
s3_crash_status_file_name))
s3_client.upload_fileobj(
bucket=os.environ.get("YAML_S3_BUCKET", ''),
s3_key=s3_key,
fileobj=io.BytesIO(json_log.encode()),
s3_kms_extra_args=utils.get_s3_extra_args())
LOG.info(
"simapp_exit_gracefully - Successfully uploaded simapp status to \
s3 bucket {} with s3 key {}.".format(
os.environ.get("YAML_S3_BUCKET", ''), s3_key))
except Exception as ex:
LOG.error("simapp_exit_gracefully - upload to s3 failed=%s", ex)
def _save_simtrace_mp4(self):
"""Get the appropriate kms key and save the simtrace and mp4 files."""
# TODO: It might be theorically possible to have different kms keys for simtrace and mp4
# but we are using the same key now since that's what happens in real life
# consider refactor the simtrace_video_s3_writers later.
if hasattr(self._current_racer.outputMp4, "s3KmsKeyArn"):
simtrace_mp4_kms = self._current_racer.outputMp4.s3KmsKeyArn
elif hasattr(self._current_racer.outputSimTrace, "s3KmsKeyArn"):
simtrace_mp4_kms = self._current_racer.outputSimTrace.s3KmsKeyArn
else:
simtrace_mp4_kms = None
for s3_writer in self._simtrace_video_s3_writers:
s3_writer.persist(utils.get_s3_extra_args(simtrace_mp4_kms))