def start_training():
"""Starts the training asynchronously using the flask executor
It runs the training based on the DSI_EXECUTE_ON environment variable and at the end,
removes the future from the executor
"""
logging.getLogger(__name__).info("Training execution started...")
# noinspection PyBroadException
try:
environment = execution_environment()
if environment == DSI_EXECUTE_ON_LOCAL:
if dvc_remote():
train(dvc_data_repo=dvc_remote(),
dvc_ssh_user=ssh_username(),
dvc_ssh_password=ssh_password())
else:
train()
elif environment == DSI_EXECUTE_ON_SSH:
connection = SSHRemoteExecutor(host=ssh_host(),
username=ssh_username(),
password=ssh_password(),
debug_mode=debug_mode()
or flask_args.debug,
port=ssh_port(),
dvc_remote=dvc_remote())
connection.setup_prerequisites()
connection.run_training()
connection.save_model_locally()
else:
raise Exception("{0} has a unknown value '{1}'".format(
DSI_EXECUTE_ON, environment))
logging.getLogger(__name__).info("Training execution ended!!!")
except Exception as training_exc:
# This exception is broad because we cannot forseen all possible exceptions in
# the DS train code.
# Also, since this train is beeing executed in a separed thread all exceptions
# should be catched
logging.getLogger(__name__).info(
"Training execution raised an exception...")
f = io.StringIO()
traceback.print_exc(file=f)
f.seek(0)
logging.getLogger(__name__).error(f.read())
raise ValueError(training_exc)
def initialize_logging(path, debug=debug_mode(), remote=False) -> None:
"""Initializes the python's logging factory with a console and a file log handler for all
log messages
Parameters
----------
path: str
Path where the logging.FileHandler should save the log messages
debug: bool
Changes the log level from INFO to DEBUG
remote: bool
If trues adds a 'REMOTE' to all log messages. Should be true only on the remote
executable scripts.
Warnings
--------
remote: Should be true only on the remote executable scripts.
"""
global log_path
log_path = path
log_formatter = logging.Formatter(
"{0}%(asctime)s [%(name)-15.15s] [%(levelname)-5.5s] %(message)s".format(
"|REMOTE| " if remote else "")
)
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG if debug else logging.INFO)
console_handler = logging.StreamHandler()
console_handler.setFormatter(log_formatter)
console_handler.setLevel(logging.DEBUG if debug else logging.INFO)
root_logger.addHandler(console_handler)
file_handler = logging.FileHandler(path)
file_handler.setFormatter(log_formatter)
file_handler.setLevel(logging.DEBUG)
file_handler.addFilter(FileHandlerFilter())
root_logger.addHandler(file_handler)
parser = argparse.ArgumentParser(
description="Training model and saving it")
parser.add_argument("--port",
"-p",
required=False,
default=8080,
type=int,
help="Port number for the Flask server")
parser.add_argument("--debug",
"-d",
action="store_true",
help="Enables debug mode in the Flask server")
parser.add_argument(
"--local",
"-l",
required=False,
default=False,
type=bool,
help="setting dummy user name and password for local development")
flask_args = parser.parse_args()
initialize_logging("prediction.log")
if flask_args.local:
username = "******"
password = "******"
app.config['USERS'][username] = password
app.run('0.0.0.0', flask_args.port, debug=debug_mode() or flask_args.debug)
def filter(self, record: logging.LogRecord) -> bool:
return not record.name.startswith(('flask', 'werkzeug')) or debug_mode()