def _on_error(ws, error):
if isinstance(error, (KeyboardInterrupt, SystemExit)):
logger.info(
"Quitting... The session will be running in the background.")
else:
logger.debug("Termination cause: %s", error)
logger.debug("Session disconnected.")
def get_files_in_path(path: str) -> List[str]:
result_files = []
for root, dirs, files in os.walk(path):
logger.debug("Root:%s, Dirs:%s", root, dirs)
for file_name in files:
result_files.append(os.path.join(root, file_name))
return result_files
def get_filename():
if is_notebook():
return "notebook"
try:
return os.path.basename(__file__)
except Exception as e:
logger.debug("Could not detect filename, %s", e)
return "not found"
def get_config(cls, check: bool = True):
if check and not cls.is_initialized():
return None
config_filepath = cls.get_config_filepath()
logger.debug("Reading config `%s` from path: %s\n", cls.__name__,
config_filepath)
return cls.read_from_path(config_filepath)
def clean_run(self, run_uuid: str, run_kind: str):
try:
self.spawner.stop(run_uuid=run_uuid, run_kind=run_kind)
except ApiException as e:
if e.status == 404:
logger.debug("Run does not exist.")
except Exception as e:
logger.debug("Run could not be cleaned: {}\n{}".format(
repr(e), traceback.format_exc()))
def get_packages():
try:
installed_packages = [d for d in pkg_resources.working_set] # noqa
return sorted(
["{}=={}".format(pkg.key, pkg.version) for pkg in installed_packages]
)
except Exception as e:
logger.debug("Could not detect installed packages, %s", e)
return []
def create_custom_object(self, name, group, version, plural, body):
resp = self.k8s_custom_object_api.create_namespaced_custom_object(
group=group,
version=version,
plural=plural,
namespace=self.namespace,
body=body,
)
logger.debug("Custom object `{}` was created".format(name))
return resp
def get_files_in_path(path: str, exclude: List[str] = None) -> List[str]:
result_files = []
for root, dirs, files in os.walk(path, topdown=True):
exclude = to_list(exclude, check_none=True)
if exclude:
dirs[:] = [d for d in dirs if d not in exclude]
logger.debug("Root:%s, Dirs:%s", root, dirs)
for file_name in files:
result_files.append(os.path.join(root, file_name))
return result_files
def delete_custom_object(self, name, group, version, plural):
self.k8s_custom_object_api.delete_namespaced_custom_object(
name=name,
group=group,
version=version,
plural=plural,
namespace=self.namespace,
body=client.V1DeleteOptions(),
)
logger.debug("Custom object `{}` deleted".format(name))
def _get_rel_asset_path(self, path: str = None, rel_path: str = None):
if not path or rel_path:
return rel_path
if hasattr(self, "_artifacts_path"):
try:
return os.path.relpath(path, self._artifacts_path)
except Exception as e:
logger.debug("could not calculate relative path %s", e)
return rel_path or path
def get_rel_asset_path(path: str = None, rel_path: str = None):
if not path or rel_path:
return rel_path
if CONTEXT_MOUNT_ARTIFACTS in path:
try:
return os.path.relpath(path, CONTEXT_MOUNT_ARTIFACTS)
except Exception as e:
logger.debug("could not calculate relative path %s", e)
return rel_path or path
async def update_custom_object(self, name, group, version, plural, body):
resp = await self.k8s_custom_object_api.patch_namespaced_custom_object(
name=name,
group=group,
version=version,
plural=plural,
namespace=self.namespace,
body=body,
)
logger.debug("Custom object `{}` was patched".format(name))
return resp
def queue_ws_request(self, request, url, **kwargs):
try:
request(url=url, **kwargs)
except Exception as e:
self._periodic_ws_exceptions += 1
logger.debug(
"Error making request url: %s, params: params %s, exp: %s",
url,
kwargs,
e,
)
finally:
self._periodic_ws_done += 1
def queue_request(self, request, url, **kwargs):
try:
request(url=url, session=self.retry_session, **kwargs)
except Exception as e:
self._threaded_exceptions += 1
logger.debug(
"Error making request url: %s, params: params %s, exp: %s",
url,
kwargs,
e,
)
finally:
self._threaded_done += 1
def delete_volume(self, name, reraise=False):
try:
self.k8s_api.delete_persistent_volume(
name=name,
body=client.V1DeleteOptions(
api_version=constants.K8S_API_VERSION_V1),
)
logger.debug("Volume `{}` Deleted".format(name))
except ApiException as e:
if reraise:
raise PolyaxonK8SError("Connection error: %s" % e) from e
else:
logger.debug("Volume `{}` was not found".format(name))
def delete_job(self, name, reraise=False):
try:
self.k8s_batch_api.delete_namespaced_job(
name=name,
namespace=self.namespace,
body=client.V1DeleteOptions(
api_version=constants.K8S_API_VERSION_V1),
)
logger.debug("Pod `{}` deleted".format(name))
except ApiException as e:
if reraise:
raise PolyaxonK8SError("Connection error: %s" % e) from e
else:
logger.debug("Pod `{}` was not found".format(name))
async def delete_custom_object(self, name, group, version, plural, reraise=False):
try:
await self.k8s_custom_object_api.delete_namespaced_custom_object(
name=name,
group=group,
version=version,
plural=plural,
namespace=self.namespace,
body=client.V1DeleteOptions(),
)
logger.debug("Custom object `{}` deleted".format(name))
except ApiException as e:
if reraise:
raise PolyaxonK8SError("Connection error: %s" % e) from e
else:
logger.debug("Custom object `{}` was not found".format(name))
def delete_deployment(self, name, reraise=False):
try:
self.k8s_apps_api.delete_namespaced_deployment(
name=name,
namespace=self.namespace,
body=client.V1DeleteOptions(
api_version=constants.K8S_API_VERSION_APPS_V1,
propagation_policy="Foreground",
),
)
logger.debug("Deployment `{}` deleted".format(name))
except ApiException as e:
if reraise:
raise PolyaxonK8SError("Connection error: %s" % e) from e
else:
logger.debug("Deployment `{}` was not found".format(name))
def delete_ingress(self, name, reraise=False):
try:
self.networking_v1_beta1_api.delete_namespaced_ingress(
name=name,
namespace=self.namespace,
body=client.V1DeleteOptions(
api_version=constants.K8S_API_VERSION_NETWORKING_V1_BETA1,
propagation_policy="Foreground",
),
)
logger.debug("Ingress `{}` deleted".format(name))
except ApiException as e:
if reraise:
raise PolyaxonK8SError("Connection error: %s" % e) from e
else:
logger.debug("Ingress `{}` was not found".format(name))
def get_files_in_current_directory(path):
"""
Gets all the files under a certain path.
Args:
path: `str`. The path to traverse for collecting files.
Returns:
list of files collected under the path.
"""
result_files = []
for root, dirs, files in os.walk(path):
logger.debug("Root:%s, Dirs:%s", root, dirs)
for file_name in files:
result_files.append(os.path.join(root, file_name))
yield result_files
def start(self) -> None:
try:
with exit_context() as exit_event:
index = 0
workers = get_pool_workers()
with ThreadPoolExecutor(workers) as pool:
logger.debug("Thread pool Workers: {}".format(workers))
timeout = self.sleep_interval or get_wait(index)
while not exit_event.wait(timeout=timeout):
index += 1
agent_state = self.process(pool)
self._check_status(agent_state)
if agent_state.state.full:
index = 2
self.ping()
timeout = self.sleep_interval or get_wait(index)
logger.info("Sleeping for {} seconds".format(timeout))
finally:
self.end()
def wrapper(*args, **kwargs):
if check_no_op and settings.CLIENT_CONFIG.no_op:
logger.debug("Using NO_OP mode")
return None
if check_offline:
self_arg = args[0] if args else None
if settings.CLIENT_CONFIG.is_offline or getattr(
self_arg, "_is_offline", False):
logger.debug("Using IS_OFFLINE mode")
return None
if args:
self_arg = args[0]
if can_log_events and (not hasattr(self_arg, "_event_logger")
or self_arg._event_logger is None # pylint:disable=protected-access
):
logger.warning(
"You should set an event logger before calling: {}".
format(f.__name__))
if can_log_outputs and (not hasattr(self_arg, "_outputs_path")
or self_arg._outputs_path is None # pylint:disable=protected-access
):
logger.warning(
"You should set an an outputs path before calling: {}".
format(f.__name__))
try:
return f(*args, **kwargs)
except (ApiException, HTTPError) as e:
logger.debug("Client config:\n%s\n",
settings.CLIENT_CONFIG.to_dict())
raise e
def get_run_env():
import pkg_resources
def get_packages():
try:
installed_packages = [d for d in pkg_resources.working_set] # noqa
return sorted([
"{}=={}".format(pkg.key, pkg.version)
for pkg in installed_packages
])
except Exception as e:
logger.debug("Could not detect installed packages, %s", e)
return []
try:
version = pkg_resources.get_distribution("polyaxon").version
except pkg_resources.DistributionNotFound:
version = ""
try:
user = getpass.getuser()
except Exception as e:
logger.debug("Could not detect installed packages, %s", e)
user = "******"
return {
"pid": os.getpid(),
"hostname": socket.gethostname(),
"os": platform.platform(aliased=True),
"system": platform.system(),
"python_version_verbose": sys.version,
"python_version": platform.python_version(),
"user": user,
"client_version": version,
"sys.argv": sys.argv,
"is_notebook": is_notebook(),
"filename": get_filename(),
"module_path": get_module_path(),
"packages": get_packages(),
}
def set_config(cls, config, init=False, visibility=None):
config_filepath = cls.get_config_filepath(visibility=visibility)
if os.path.isfile(config_filepath) and init:
logger.debug("%s file already present at %s", cls.CONFIG_FILE_NAME,
config_filepath)
return
with open(config_filepath, "w") as config_file:
if hasattr(config, "to_dict"):
logger.debug("Setting %s in the file %s", config.to_dict(),
cls.CONFIG_FILE_NAME)
config_file.write(ujson.dumps(config.to_dict()))
elif isinstance(config, Mapping):
config_file.write(ujson.dumps(config))
else:
logger.debug("Setting %s in the file %s", config,
cls.CONFIG_FILE_NAME)
config_file.write(config)
def get_unignored_filepaths(cls):
config = cls.get_config()
unignored_files = []
for root, dirs, files in os.walk("."):
logger.debug("Root:%s, Dirs:%s", root, dirs)
if cls.is_ignored(unix_style_path(root), config):
dirs[:] = []
logger.debug("Ignoring directory : %s", root)
continue
for file_name in files:
filepath = unix_style_path(os.path.join(root, file_name))
if cls.is_ignored(filepath, config):
logger.debug("Ignoring file : %s", file_name)
continue
unignored_files.append(os.path.join(root, file_name))
return unignored_files
def request(
self,
method,
url,
params=None,
data=None,
files=None,
json=None, # noqa
timeout=None,
headers=None,
session=None,
):
"""Send a request with the given data as json to the given URL.
Args:
method: HTTP method
url: The url to send the request to.
params: Dictionary of values to format url.
data:
files:
json:
timeout:
headers: extra headers to add to the request.
Returns:
Request response if successful, Exception otherwise.
Raises:
PolyaxonHTTPError or one of its subclasses.
"""
logger.debug("Starting request to url: %s with params: %s, data: %s",
url, params, data)
request_headers = self._get_headers(headers=headers)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
session = session or self.session
try:
response = session.request(
method,
url,
params=params,
data=data,
json=json,
headers=request_headers,
files=files,
timeout=timeout,
verify=self.config.verify_ssl,
)
except (
requests.exceptions.RequestException,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as exception:
try:
logger.debug("Exception: %s", exception, exc_info=True)
except TypeError:
pass
raise PolyaxonShouldExitError(
"Error connecting to Polyaxon server on `{}`.\n"
"An Error `{}` occurred.\n"
"Check your host and ports configuration "
"and your internet connection.".format(url, exception))
logger.debug("Response Content: %s, Headers: %s", response.content,
response.headers)
self.check_response_status(response, url)
return response
def download(
self,
url,
filename,
params=None,
headers=None,
timeout=None,
session=None,
untar=False,
delete_tar=True,
extract_path=None,
):
"""
Download the file from the given url at the current path
"""
# pylint:disable=too-many-branches
logger.debug("Downloading files from url: %s", url)
request_headers = self._get_headers(headers=headers)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
session = session or self.session
try:
response = session.get(
url,
params=params,
headers=request_headers,
timeout=timeout,
stream=True,
)
self.check_response_status(response, url)
with open(filename, "wb") as f:
# chunk mode response doesn't have content-length so we are
# using a custom header here
content_length = response.headers.get(
"x-polyaxon-content-length")
if not content_length:
content_length = response.headers.get("content-length")
if content_length:
for chunk in progress_bar(
response.iter_content(chunk_size=1024),
expected_size=(int(content_length) / 1024) + 1,
):
if chunk:
f.write(chunk)
else:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
if untar:
filename = self.untar(filename=filename,
delete_tar=delete_tar,
extract_path=extract_path)
return filename
except (
requests.exceptions.ConnectionError,
requests.exceptions.RequestException,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as exception:
try:
logger.debug("Exception: %s", exception)
except TypeError:
pass
raise PolyaxonShouldExitError(
"Error connecting to Polyaxon server on `{}`.\n"
"An Error `{}` occurred.\n"
"Check your host and ports configuration "
"and your internet connection.".format(url, exception))
def get_module_path():
try:
return os.path.dirname(os.path.realpath("__file__"))
except Exception as e:
logger.debug("Could not detect module path, %s", e)
return "not found"
def create_ingress(self, name, body):
resp = self.networking_v1_beta1_api.create_namespaced_ingress(
namespace=self.namespace, body=body)
logger.debug("ingress `{}` was created".format(name))
return resp
def update_ingress(self, name, body):
resp = self.networking_v1_beta1_api.patch_namespaced_ingress(
name=name, namespace=self.namespace, body=body)
logger.debug("Ingress `{}` was patched".format(name))
return resp
