def download(self, url, filename, relative=False, headers=None, timeout=5):
"""
Download the file from the given url at the current path
"""
request_url = self.base_url + url if relative else url
logger.debug("Downloading file from url: {}".format(request_url))
request_headers = self._get_headers(headers=headers)
try:
response = requests.get(request_url,
headers=request_headers,
timeout=timeout,
stream=True)
self.check_response_status(response, request_url)
with open(filename, 'wb') as f:
# chunk mode response doesn't have content-length so we are
# using a custom header here
content_length = response.headers.get('x-polyaxon-content-length')
if not content_length:
content_length = response.headers.get('content-length')
if content_length:
for chunk in progress.bar(response.iter_content(chunk_size=1024),
expected_size=(int(content_length) / 1024) + 1):
if chunk:
f.write(chunk)
else:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return filename
except requests.exceptions.ConnectionError as exception:
logger.debug("Exception: {}".format(exception))
raise PolyaxonShouldExitError(
"Cannot connect to the Polyaxon server. Check your internet connection.")
def get_filename():
if is_notebook():
return 'notebook'
try:
return os.path.basename(__file__)
except Exception as e:
logger.debug('Could not detect filename, %s', e)
return 'not found'
def queue_ws_request(self, request, url, **kwargs):
try:
request(url=url, **kwargs)
except Exception as e:
self._periodic_ws_exceptions += 1
logger.debug(
'Error making request url: %s, params: params %s, exp: %s',
url, kwargs, e)
finally:
self._periodic_ws_done += 1
def queue_request(self, request, url, **kwargs):
try:
request(url=url, session=self.retry_session, **kwargs)
except Exception as e:
self._threaded_exceptions += 1
logger.debug(
'Error making request url: %s, params: params %s, exp: %s',
url, kwargs, e)
finally:
self._threaded_done += 1
def get_packages():
try:
installed_packages = [d for d in pkg_resources.working_set] # noqa
return sorted([
"{}=={}".format(pkg.key, pkg.version)
for pkg in installed_packages
])
except Exception as e:
logger.debug('Could not detect installed packages, %s', e)
return []
def __init__(self,
project=None,
experiment_id=None,
group_id=None,
client=None,
track_logs=True,
track_code=True,
track_env=True,
outputs_store=None):
if settings.NO_OP:
return
if project is None and settings.IS_MANAGED and not self.is_notebook_job:
experiment_info = self.get_experiment_info()
project = experiment_info['project_name']
experiment_id = experiment_info['experiment_name'].split('.')[-1]
super(Experiment, self).__init__(project=project,
client=client,
track_logs=track_logs,
track_code=track_code,
track_env=track_env,
outputs_store=outputs_store)
self.experiment_id = experiment_id
self.group_id = group_id
self.experiment = None
if settings.IS_OFFLINE:
return
# Check if there's an ephemeral token
check_ephemeral_token = (settings.IS_MANAGED and
hasattr(settings, 'SECRET_EPHEMERAL_TOKEN') and
settings.SECRET_EPHEMERAL_TOKEN)
if check_ephemeral_token:
try:
self.client.auth.login_experiment_ephemeral_token(
username=self.username,
project_name=self.project_name,
experiment_id=self.experiment_id,
ephemeral_token=settings.SECRET_EPHEMERAL_TOKEN,
set_token=True,
persist_token=True)
except AuthenticationError:
logger.debug('Could not log with ephemeral token.')
if settings.IS_MANAGED:
self._set_health_url()
# Track run env
if settings.IS_MANAGED and self.track_env and not self.is_notebook_job:
self.log_run_env()
def download(self, url, filename, headers=None, timeout=None, session=None):
"""
Download the file from the given url at the current path
"""
logger.debug("Downloading files from url: %s", url)
request_headers = self._get_headers(headers=headers)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
session = session or self.session
try:
response = session.get(url,
headers=request_headers,
timeout=timeout,
stream=True)
self.check_response_status(response, url)
with open(filename, 'wb') as f:
# chunk mode response doesn't have content-length so we are
# using a custom header here
content_length = response.headers.get('x-polyaxon-content-length')
if not content_length:
content_length = response.headers.get('content-length')
if content_length:
for chunk in progress.bar(response.iter_content(chunk_size=1024),
expected_size=(int(content_length) / 1024) + 1):
if chunk:
f.write(chunk)
else:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return filename
except (requests.exceptions.ConnectionError,
requests.exceptions.RequestException,
requests.exceptions.Timeout,
requests.exceptions.HTTPError) as exception:
try:
logger.debug("Exception: %s", exception)
except TypeError:
pass
raise PolyaxonShouldExitError(
"Error connecting to Polyaxon server on `{}`.\n"
"An Error `{}` occurred.\n"
"Check your host and ports configuration "
"and your internet connection.".format(url, exception))
def get_run_env():
import pkg_resources
def get_packages():
try:
installed_packages = [d for d in pkg_resources.working_set] # noqa
return sorted([
"{}=={}".format(pkg.key, pkg.version)
for pkg in installed_packages
])
except Exception as e:
logger.debug('Could not detect installed packages, %s', e)
return []
try:
version = pkg_resources.get_distribution(PROJECT_CLIENT_NAME).version
except pkg_resources.DistributionNotFound:
version = ''
try:
user = getpass.getuser()
except Exception as e:
logger.debug('Could not detect installed packages, %s', e)
user = '******'
return {
'pid': os.getpid(),
'hostname': socket.gethostname(),
'os': platform.platform(aliased=True),
'system': platform.system(),
'python_version_verbose': sys.version,
'python_version': platform.python_version(),
'user': user,
'client_version': version,
'sys.argv': sys.argv,
'is_notebook': is_notebook(),
'filename': get_filename(),
'module_path': get_module_path(),
'packages': get_packages(),
}
def request(self,
method,
url,
params=None,
data=None,
files=None,
json=None,
timeout=5,
headers=None):
"""Send a request with the given data as json to the given URL.
Args:
method: HTTP method
url: The url to send the request to.
params: Dictionary of values to format url.
data:
files:
json:
timeout:
headers: extra headers to add to the request.
Returns:
Request response if successful, Exception otherwise.
Raises:
PolyaxonHTTPError or one of its subclasses.
"""
logger.debug("Starting request to url: {} with params: {}, data: {}".format(
url, params, data))
request_headers = self._get_headers(headers=headers)
try:
response = requests.request(method,
url,
params=params,
data=data,
json=json,
headers=request_headers,
files=files,
timeout=timeout)
except requests.exceptions.ConnectionError as exception:
logger.debug("Exception: %s", exception, exc_info=True)
raise PolyaxonShouldExitError(
"Cannot connect to the Polyaxon server on `{}`.\n"
"Check your host and ports configuration and your internet connection.".format(url))
logger.debug("Response Content: %s, Headers: %s" % (response.content, response.headers))
self.check_response_status(response, url)
return response
def _on_error(ws, error):
if isinstance(error, (KeyboardInterrupt, SystemExit)):
logger.info('Quitting... The session will be running in the background.')
else:
logger.debug('Termination cause: %s', error)
logger.debug('Session disconnected.')
def request(
self,
method,
url,
params=None,
data=None,
files=None,
json=None, # noqa
timeout=None,
headers=None,
session=None):
"""Send a request with the given data as json to the given URL.
Args:
method: HTTP method
url: The url to send the request to.
params: Dictionary of values to format url.
data:
files:
json:
timeout:
headers: extra headers to add to the request.
Returns:
Request response if successful, Exception otherwise.
Raises:
PolyaxonHTTPError or one of its subclasses.
"""
logger.debug("Starting request to url: %s with params: %s, data: %s",
url, params, data)
request_headers = self._get_headers(headers=headers)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
session = session or self.session
try:
response = session.request(method,
url,
params=params,
data=data,
json=json,
headers=request_headers,
files=files,
timeout=timeout,
verify=self.config.verify_ssl)
except (requests.exceptions.RequestException,
requests.exceptions.Timeout,
requests.exceptions.HTTPError) as exception:
try:
logger.debug("Exception: %s", exception, exc_info=True)
except TypeError:
pass
raise PolyaxonShouldExitError(
"Error connecting to Polyaxon server on `{}`.\n"
"An Error `{}` occurred.\n"
"Check your host and ports configuration "
"and your internet connection.".format(url, exception))
logger.debug("Response Content: %s, Headers: %s", response.content,
response.headers)
self.check_response_status(response, url)
return response
def get_module_path():
try:
return os.path.dirname(os.path.realpath('__file__'))
except Exception as e:
logger.debug('Could not detect module path, %s', e)
return 'not found'
