def load_remote_model(train_pod_url, git_commit): """Helper function for loading a model from the remote training service. Using the *getmodel* endpoint of the training service. Parameters ---------- train_pod_url : String url of the training service git_commit : String Returns ------- predictor : ModelWrapper object Wrapped algorithm in the ModelWrapper class """ auth = training_auth() if not auth: return None username, password = auth.popitem() try: response = requests.get('{0}/getmodel'.format(train_pod_url), auth=HTTPBasicAuth(username=username, password=password), stream=True) if response.status_code == 200: with open(git_commit, 'wb') as f: f.write(response.content) predictor = joblib.load(git_commit) print("model loaded") else: print("/getmodel endpoint from training pod returned: {0}".format( response.content)) return None return predictor except (ConnectionError, ConnectionRefusedError, ConnectionAbortedError, ConnectionResetError, RequestException): print("remote training pod not reachable") return None
DSI_EXECUTE_ON_LOCAL, DSI_EXECUTE_ON_SSH, ssh_host, \ ssh_username, ssh_password, ssh_port, DSI_EXECUTE_ON, training_auth, dvc_remote from services.infrastructure.flask import init_flask, status from services.infrastructure.git_info import GIT_COMMIT, GIT_COMMIT_SHORT, GIT_BRANCH, \ GIT_LAST_CHANGE, GIT_REPO_NAME from services.infrastructure.logging import initialize_logging from services.infrastructure.remote.ssh.executors import SSHRemoteExecutor TRAINING_KEY = '_training-job-key_' """ Key used for storing the training thread in the executor. It ensures that only one training instance will run at certain point in time """ app, _executor, auth = init_flask() app.config['USERS'] = training_auth() @_executor.job @auth.login_required def start_training(): """Starts the training asynchronously using the flask executor It runs the training based on the DSI_EXECUTE_ON environment variable and at the end, removes the future from the executor """ logging.getLogger(__name__).info("Training execution started...") # noinspection PyBroadException try: environment = execution_environment() if environment == DSI_EXECUTE_ON_LOCAL: