def __init__(self, transform_fn, initialize_fn=None, module_name=None, healthcheck_fn=None): """Creates and Flask application from a transformer. Args: transform_fn (function): responsible to make predictions against the model. Follows the signature: * Returns: `sagemaker_containers.transformers.TransformSpec`: named tuple with prediction data. initialize_fn (function, optional): this function is called when the Flask application starts. It doest not have return type or arguments. healthcheck_fn (function, optional): function that will be used for healthcheck calls when the containers starts, if not specified, it will use ping as the default healthcheck call. Signature: * Returns: `flask.app.Response`: response object with new healthcheck response. module_name (str): the module name which implements the worker. If not specified, it will use sagemaker_containers.ServingEnv().module_name as the default module name. """ super(Worker, self).__init__(module_name or env.module_name) # the logger is configured after importing the framework library, allowing the framework to # configure logging at import time. _logging.configure_logger(env.log_level) if initialize_fn: self.before_first_request(initialize_fn) self.add_url_rule(rule="/invocations", endpoint="invocations", view_func=transform_fn, methods=["POST"]) self.add_url_rule(rule="/ping", endpoint="ping", view_func=healthcheck_fn or default_healthcheck_fn) self.request_class = Request
def train(): try: # TODO: iquintero - add error handling for ImportError to let the user know # if the framework module is not defined. env = sagemaker_containers.training_env() framework_name, entry_point_name = env.framework_module.split(':') try: framework = importlib.import_module(framework_name) except: logger.error("Import failure loading %s. sys.path=%s" % (framework_name, sys.path)) raise # the logger is configured after importing the framework library, allowing the framework to # configure logging at import time. _logging.configure_logger(env.log_level) logger.info('Imported framework %s', framework_name) entry_point = getattr(framework, entry_point_name) _modules.write_env_vars(env.to_env_vars()) entry_point() logger.info('Reporting training SUCCESS') _files.write_success_file() _exit_processes(SUCCESS_CODE) except _errors.ClientError as e: failure_message = str(e) _files.write_failure_file(failure_message) logger.error(failure_message) _exit_processes(DEFAULT_FAILURE_CODE) except Exception as e: failure_msg = 'framework error: \n%s\n%s' % (traceback.format_exc(), str(e)) _files.write_failure_file(failure_msg) logger.error('Reporting training FAILURE') logger.error(failure_msg) exit_code = getattr(e, 'errno', DEFAULT_FAILURE_CODE) _exit_processes(exit_code)
def train(): intermediate_sync = None exit_code = SUCCESS_CODE try: # TODO: iquintero - add error handling for ImportError to let the user know # if the framework module is not defined. env = sagemaker_containers.training_env() # TODO: [issue#144] There is a bug in the logic - # we need os.environ.get(_params.REGION_NAME_ENV) # in certain regions, but it is not going to be available unless # TrainingEnvironment has been initialized. It shouldn't be environment variable. region = os.environ.get('AWS_REGION', os.environ.get(_params.REGION_NAME_ENV)) s3_endpoint_url = os.environ.get("S3_ENDPOINT_URL") intermediate_sync = _intermediate_output.start_sync(env.sagemaker_s3_output(), region, endpoint_url=s3_endpoint_url) if env.framework_module: framework_name, entry_point_name = env.framework_module.split(':') framework = importlib.import_module(framework_name) # the logger is configured after importing the framework library, allowing the framework to # configure logging at import time. _logging.configure_logger(env.log_level) logger.info('Imported framework %s', framework_name) entrypoint = getattr(framework, entry_point_name) entrypoint() else: _logging.configure_logger(env.log_level) mpi_enabled = env.additional_framework_parameters.get(_params.MPI_ENABLED) runner_type = _runner.RunnerType.MPI if mpi_enabled else _runner.RunnerType.Process entry_point.run(env.module_dir, env.user_entry_point, env.to_cmd_args(), env.to_env_vars(), runner=runner_type) logger.info('Reporting training SUCCESS') _files.write_success_file() except _errors.ClientError as e: failure_message = str(e) _files.write_failure_file(failure_message) logger.error(failure_message) if intermediate_sync: intermediate_sync.join() exit_code = DEFAULT_FAILURE_CODE except Exception as e: failure_msg = 'framework error: \n%s\n%s' % (traceback.format_exc(), str(e)) _files.write_failure_file(failure_msg) logger.error('Reporting training FAILURE') logger.error(failure_msg) exit_code = getattr(e, 'errno', DEFAULT_FAILURE_CODE) finally: if intermediate_sync: intermediate_sync.join() _exit_processes(exit_code)
def train(): """Placeholder docstring""" intermediate_sync = None exit_code = SUCCESS_CODE try: env = sagemaker_containers.training_env() region = os.environ.get("AWS_REGION", os.environ.get(_params.REGION_NAME_ENV)) s3_endpoint_url = os.environ.get(_params.S3_ENDPOINT_URL, None) intermediate_sync = _intermediate_output.start_sync( env.sagemaker_s3_output(), region, endpoint_url=s3_endpoint_url) if env.framework_module: framework_name, entry_point_name = env.framework_module.split(":") framework = importlib.import_module(framework_name) # the logger is configured after importing the framework library, allowing # the framework to configure logging at import time. _logging.configure_logger(env.log_level) logger.info("Imported framework %s", framework_name) entrypoint = getattr(framework, entry_point_name) entrypoint() else: _logging.configure_logger(env.log_level) mpi_enabled = env.additional_framework_parameters.get( _params.MPI_ENABLED) runner_type = _runner.RunnerType.MPI if mpi_enabled else _runner.RunnerType.Process entry_point.run( env.module_dir, env.user_entry_point, env.to_cmd_args(), env.to_env_vars(), runner=runner_type, ) logger.info("Reporting training SUCCESS") _files.write_success_file() except _errors.ClientError as e: failure_message = str(e) _files.write_failure_file(failure_message) logger.error(failure_message) if intermediate_sync: intermediate_sync.join() exit_code = DEFAULT_FAILURE_CODE except Exception as e: # pylint: disable=broad-except failure_msg = "framework error: \n%s\n%s" % (traceback.format_exc(), str(e)) _files.write_failure_file(failure_msg) logger.error("Reporting training FAILURE") logger.error(failure_msg) error_number = getattr(e, "errno", DEFAULT_FAILURE_CODE) exit_code = _get_valid_failure_exit_code(error_number) finally: if intermediate_sync: intermediate_sync.join() _exit_processes(exit_code)