def load_remote_model(train_pod_url, git_commit):
"""Helper function for loading a model from the remote training service. Using the *getmodel*
endpoint of the training service.
Parameters
----------
train_pod_url : String
url of the training service
git_commit : String
Returns
-------
predictor : ModelWrapper object
Wrapped algorithm in the ModelWrapper class
"""
auth = training_auth()
if not auth:
return None
username, password = auth.popitem()
try:
response = requests.get('{0}/getmodel'.format(train_pod_url),
auth=HTTPBasicAuth(username=username,
password=password),
stream=True)
if response.status_code == 200:
with open(git_commit, 'wb') as f:
f.write(response.content)
predictor = joblib.load(git_commit)
print("model loaded")
else:
print("/getmodel endpoint from training pod returned: {0}".format(
response.content))
return None
return predictor
except (ConnectionError, ConnectionRefusedError, ConnectionAbortedError,
ConnectionResetError, RequestException):
print("remote training pod not reachable")
return None
DSI_EXECUTE_ON_LOCAL, DSI_EXECUTE_ON_SSH, ssh_host, \
ssh_username, ssh_password, ssh_port, DSI_EXECUTE_ON, training_auth, dvc_remote
from services.infrastructure.flask import init_flask, status
from services.infrastructure.git_info import GIT_COMMIT, GIT_COMMIT_SHORT, GIT_BRANCH, \
GIT_LAST_CHANGE, GIT_REPO_NAME
from services.infrastructure.logging import initialize_logging
from services.infrastructure.remote.ssh.executors import SSHRemoteExecutor
TRAINING_KEY = '_training-job-key_'
"""
Key used for storing the training thread in the executor. It ensures that only one training
instance will run at certain point in time
"""
app, _executor, auth = init_flask()
app.config['USERS'] = training_auth()
@_executor.job
@auth.login_required
def start_training():
"""Starts the training asynchronously using the flask executor
It runs the training based on the DSI_EXECUTE_ON environment variable and at the end,
removes the future from the executor
"""
logging.getLogger(__name__).info("Training execution started...")
# noinspection PyBroadException
try:
environment = execution_environment()
if environment == DSI_EXECUTE_ON_LOCAL: