def download(self, url, filename, relative=False, headers=None, timeout=5): """ Download the file from the given url at the current path """ request_url = self.base_url + url if relative else url logger.debug("Downloading file from url: {}".format(request_url)) request_headers = self._get_headers(headers=headers) try: response = requests.get(request_url, headers=request_headers, timeout=timeout, stream=True) self.check_response_status(response, request_url) with open(filename, 'wb') as f: # chunk mode response doesn't have content-length so we are # using a custom header here content_length = response.headers.get('x-polyaxon-content-length') if not content_length: content_length = response.headers.get('content-length') if content_length: for chunk in progress.bar(response.iter_content(chunk_size=1024), expected_size=(int(content_length) / 1024) + 1): if chunk: f.write(chunk) else: for chunk in response.iter_content(chunk_size=1024): if chunk: f.write(chunk) return filename except requests.exceptions.ConnectionError as exception: logger.debug("Exception: {}".format(exception)) raise PolyaxonShouldExitError( "Cannot connect to the Polyaxon server. Check your internet connection.")
def get_filename(): if is_notebook(): return 'notebook' try: return os.path.basename(__file__) except Exception as e: logger.debug('Could not detect filename, %s', e) return 'not found'
def queue_ws_request(self, request, url, **kwargs): try: request(url=url, **kwargs) except Exception as e: self._periodic_ws_exceptions += 1 logger.debug( 'Error making request url: %s, params: params %s, exp: %s', url, kwargs, e) finally: self._periodic_ws_done += 1
def queue_request(self, request, url, **kwargs): try: request(url=url, session=self.retry_session, **kwargs) except Exception as e: self._threaded_exceptions += 1 logger.debug( 'Error making request url: %s, params: params %s, exp: %s', url, kwargs, e) finally: self._threaded_done += 1
def get_packages(): try: installed_packages = [d for d in pkg_resources.working_set] # noqa return sorted([ "{}=={}".format(pkg.key, pkg.version) for pkg in installed_packages ]) except Exception as e: logger.debug('Could not detect installed packages, %s', e) return []
def __init__(self, project=None, experiment_id=None, group_id=None, client=None, track_logs=True, track_code=True, track_env=True, outputs_store=None): if settings.NO_OP: return if project is None and settings.IS_MANAGED and not self.is_notebook_job: experiment_info = self.get_experiment_info() project = experiment_info['project_name'] experiment_id = experiment_info['experiment_name'].split('.')[-1] super(Experiment, self).__init__(project=project, client=client, track_logs=track_logs, track_code=track_code, track_env=track_env, outputs_store=outputs_store) self.experiment_id = experiment_id self.group_id = group_id self.experiment = None if settings.IS_OFFLINE: return # Check if there's an ephemeral token check_ephemeral_token = (settings.IS_MANAGED and hasattr(settings, 'SECRET_EPHEMERAL_TOKEN') and settings.SECRET_EPHEMERAL_TOKEN) if check_ephemeral_token: try: self.client.auth.login_experiment_ephemeral_token( username=self.username, project_name=self.project_name, experiment_id=self.experiment_id, ephemeral_token=settings.SECRET_EPHEMERAL_TOKEN, set_token=True, persist_token=True) except AuthenticationError: logger.debug('Could not log with ephemeral token.') if settings.IS_MANAGED: self._set_health_url() # Track run env if settings.IS_MANAGED and self.track_env and not self.is_notebook_job: self.log_run_env()
def download(self, url, filename, headers=None, timeout=None, session=None): """ Download the file from the given url at the current path """ logger.debug("Downloading files from url: %s", url) request_headers = self._get_headers(headers=headers) timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT session = session or self.session try: response = session.get(url, headers=request_headers, timeout=timeout, stream=True) self.check_response_status(response, url) with open(filename, 'wb') as f: # chunk mode response doesn't have content-length so we are # using a custom header here content_length = response.headers.get('x-polyaxon-content-length') if not content_length: content_length = response.headers.get('content-length') if content_length: for chunk in progress.bar(response.iter_content(chunk_size=1024), expected_size=(int(content_length) / 1024) + 1): if chunk: f.write(chunk) else: for chunk in response.iter_content(chunk_size=1024): if chunk: f.write(chunk) return filename except (requests.exceptions.ConnectionError, requests.exceptions.RequestException, requests.exceptions.Timeout, requests.exceptions.HTTPError) as exception: try: logger.debug("Exception: %s", exception) except TypeError: pass raise PolyaxonShouldExitError( "Error connecting to Polyaxon server on `{}`.\n" "An Error `{}` occurred.\n" "Check your host and ports configuration " "and your internet connection.".format(url, exception))
def get_run_env(): import pkg_resources def get_packages(): try: installed_packages = [d for d in pkg_resources.working_set] # noqa return sorted([ "{}=={}".format(pkg.key, pkg.version) for pkg in installed_packages ]) except Exception as e: logger.debug('Could not detect installed packages, %s', e) return [] try: version = pkg_resources.get_distribution(PROJECT_CLIENT_NAME).version except pkg_resources.DistributionNotFound: version = '' try: user = getpass.getuser() except Exception as e: logger.debug('Could not detect installed packages, %s', e) user = '******' return { 'pid': os.getpid(), 'hostname': socket.gethostname(), 'os': platform.platform(aliased=True), 'system': platform.system(), 'python_version_verbose': sys.version, 'python_version': platform.python_version(), 'user': user, 'client_version': version, 'sys.argv': sys.argv, 'is_notebook': is_notebook(), 'filename': get_filename(), 'module_path': get_module_path(), 'packages': get_packages(), }
def request(self, method, url, params=None, data=None, files=None, json=None, timeout=5, headers=None): """Send a request with the given data as json to the given URL. Args: method: HTTP method url: The url to send the request to. params: Dictionary of values to format url. data: files: json: timeout: headers: extra headers to add to the request. Returns: Request response if successful, Exception otherwise. Raises: PolyaxonHTTPError or one of its subclasses. """ logger.debug("Starting request to url: {} with params: {}, data: {}".format( url, params, data)) request_headers = self._get_headers(headers=headers) try: response = requests.request(method, url, params=params, data=data, json=json, headers=request_headers, files=files, timeout=timeout) except requests.exceptions.ConnectionError as exception: logger.debug("Exception: %s", exception, exc_info=True) raise PolyaxonShouldExitError( "Cannot connect to the Polyaxon server on `{}`.\n" "Check your host and ports configuration and your internet connection.".format(url)) logger.debug("Response Content: %s, Headers: %s" % (response.content, response.headers)) self.check_response_status(response, url) return response
def _on_error(ws, error): if isinstance(error, (KeyboardInterrupt, SystemExit)): logger.info('Quitting... The session will be running in the background.') else: logger.debug('Termination cause: %s', error) logger.debug('Session disconnected.')
def request( self, method, url, params=None, data=None, files=None, json=None, # noqa timeout=None, headers=None, session=None): """Send a request with the given data as json to the given URL. Args: method: HTTP method url: The url to send the request to. params: Dictionary of values to format url. data: files: json: timeout: headers: extra headers to add to the request. Returns: Request response if successful, Exception otherwise. Raises: PolyaxonHTTPError or one of its subclasses. """ logger.debug("Starting request to url: %s with params: %s, data: %s", url, params, data) request_headers = self._get_headers(headers=headers) timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT session = session or self.session try: response = session.request(method, url, params=params, data=data, json=json, headers=request_headers, files=files, timeout=timeout, verify=self.config.verify_ssl) except (requests.exceptions.RequestException, requests.exceptions.Timeout, requests.exceptions.HTTPError) as exception: try: logger.debug("Exception: %s", exception, exc_info=True) except TypeError: pass raise PolyaxonShouldExitError( "Error connecting to Polyaxon server on `{}`.\n" "An Error `{}` occurred.\n" "Check your host and ports configuration " "and your internet connection.".format(url, exception)) logger.debug("Response Content: %s, Headers: %s", response.content, response.headers) self.check_response_status(response, url) return response
def get_module_path(): try: return os.path.dirname(os.path.realpath('__file__')) except Exception as e: logger.debug('Could not detect module path, %s', e) return 'not found'