def start_training():
"""Starts the training asynchronously using the flask executor
It runs the training based on the DSI_EXECUTE_ON environment variable and at the end,
removes the future from the executor
"""
logging.getLogger(__name__).info("Training execution started...")
# noinspection PyBroadException
try:
environment = execution_environment()
if environment == DSI_EXECUTE_ON_LOCAL:
if dvc_remote():
train(dvc_data_repo=dvc_remote(),
dvc_ssh_user=ssh_username(),
dvc_ssh_password=ssh_password())
else:
train()
elif environment == DSI_EXECUTE_ON_SSH:
connection = SSHRemoteExecutor(host=ssh_host(),
username=ssh_username(),
password=ssh_password(),
debug_mode=debug_mode()
or flask_args.debug,
port=ssh_port(),
dvc_remote=dvc_remote())
connection.setup_prerequisites()
connection.run_training()
connection.save_model_locally()
else:
raise Exception("{0} has a unknown value '{1}'".format(
DSI_EXECUTE_ON, environment))
logging.getLogger(__name__).info("Training execution ended!!!")
except Exception as training_exc:
# This exception is broad because we cannot forseen all possible exceptions in
# the DS train code.
# Also, since this train is beeing executed in a separed thread all exceptions
# should be catched
logging.getLogger(__name__).info(
"Training execution raised an exception...")
f = io.StringIO()
traceback.print_exc(file=f)
f.seek(0)
logging.getLogger(__name__).error(f.read())
raise ValueError(training_exc)
def initialize_logging(path, debug=debug_mode(), remote=False) -> None:
"""Initializes the python's logging factory with a console and a file log handler for all
log messages
Parameters
----------
path: str
Path where the logging.FileHandler should save the log messages
debug: bool
Changes the log level from INFO to DEBUG
remote: bool
If trues adds a 'REMOTE' to all log messages. Should be true only on the remote
executable scripts.
Warnings
--------
remote: Should be true only on the remote executable scripts.
"""
global log_path
log_path = path
log_formatter = logging.Formatter(
"{0}%(asctime)s [%(name)-15.15s] [%(levelname)-5.5s] %(message)s".format(
"|REMOTE| " if remote else "")
)
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG if debug else logging.INFO)
console_handler = logging.StreamHandler()
console_handler.setFormatter(log_formatter)
console_handler.setLevel(logging.DEBUG if debug else logging.INFO)
root_logger.addHandler(console_handler)
file_handler = logging.FileHandler(path)
file_handler.setFormatter(log_formatter)
file_handler.setLevel(logging.DEBUG)
file_handler.addFilter(FileHandlerFilter())
root_logger.addHandler(file_handler)
except KeyError as e:
msg = 'Problem with the provided json post: {0}'.format(e)
app.logger.error(msg)
resp = jsonify({'error': msg})
resp.status_code = 400
return resp
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Training model and saving it")
parser.add_argument("--port", "-p", required=False, default=8080, type=int,
help="Port number for the Flask server")
parser.add_argument("--debug", "-d", action="store_true",
help="Enables debug mode in the Flask server")
parser.add_argument("--local", "-l", required=False, default=False, type=bool,
help="setting dummy user name and password for local development")
flask_args = parser.parse_args()
initialize_logging("prediction.log")
if flask_args.local:
username = "******"
password = "******"
app.config['USERS'][username] = password
app.run('0.0.0.0', flask_args.port, debug=debug_mode() or flask_args.debug)
else:
initialize_logging(path="prediction.log", debug=os.getenv('DEBUG', False))
gunicorn_app = app
def filter(self, record: logging.LogRecord) -> bool:
return not record.name.startswith(('flask', 'werkzeug')) or debug_mode()