def _create_retry_handler(self): original_config = open(os.path.join(CLIENT_DIR, '_retry.json')).read() original_config = json.loads(original_config, object_pairs_hook=OrderedDict) config = build_retry_config(original_config['retry'], original_config.get('definitions', {})) return create_retry_handler(config)
def test_create_retry_handler_with_no_operation(self): handler = retryhandler.create_retry_handler(self.retry_config, operation_name=None) self.assertIsInstance(handler, retryhandler.RetryHandler) # No good way to test for the delay function as the action # other than to just invoke it. self.assertEqual(handler._action(attempts=2), 2) self.assertEqual(handler._action(attempts=3), 4)
def test_connection_timeouts_are_retried(self): # If a connection times out, we get a Timout exception # from requests. We should be retrying those. handler = retryhandler.create_retry_handler( self.retry_config, operation_name='OperationBar') sleep_time = handler(response=None, attempts=1, caught_exception=Timeout()) self.assertEqual(sleep_time, 1)
def _create_default_retryhandler(self): # We create one retryhandler based on the __default__ configuration in # the _retry.json (in the 'data' directory). This retryhandler is used # by all services. config = self._load_retry_config() if not config: return LOG.info("Using retry config: %s" % config) return create_retry_handler(config)
def test_retry_pool_closed_errors(self): # A ClosedPoolError is retried (this is a workaround for a urllib3 # bug). Can be removed once we upgrade to requests 2.0.0. handler = retryhandler.create_retry_handler( self.retry_config, operation_name='OperationBar') # 4th attempt is retried. sleep_time = handler(response=None, attempts=4, caught_exception=ClosedPoolError( 'FakePool', 'Message')) self.assertEqual(sleep_time, 8) # But the 5th time propogates the error. with self.assertRaises(ClosedPoolError): handler(response=None, attempts=10, caught_exception=ClosedPoolError('FakePool', 'Message'))
def test_503_are_retried(self): handler = retryhandler.create_retry_handler( self.retry_config, operation_name='OperationFoo') http_response = mock.Mock() http_response.status_code = 503 # Max retry is configurd to 5. self.assertEqual( handler(response=(http_response, {}), attempts=1, caught_exception=None), 1) self.assertEqual( handler(response=(http_response, {}), attempts=4, caught_exception=None), 8) self.assertEqual( handler(response=(http_response, {}), attempts=5, caught_exception=None), None)
def test_create_retry_handler_with_socket_errors(self): handler = retryhandler.create_retry_handler( self.retry_config, operation_name='OperationBar') with self.assertRaises(ConnectionError): handler(response=None, attempts=10, caught_exception=ConnectionError()) # No connection error raised because attempts < max_attempts. sleep_time = handler(response=None, attempts=1, caught_exception=ConnectionError()) self.assertEqual(sleep_time, 1) # But any other exception should be raised even if # attempts < max_attempts. with self.assertRaises(ValueError): sleep_time = handler(response=None, attempts=1, caught_exception=ValueError())
def test_bogus_configuration_raises_exception(self): with self.assertRaises(ValueError): retryhandler.create_retry_handler(self.retry_config, operation_name='OperationBogus')
def __init__(self, debug=False, tls_verify=False, strict_errors=False, tls_warnings=False, client_endpoint=None, cdp_credentials=None, error_handler=None, warning_handler=None, scrub_inputs=True, cp_region='default', agent_header=None): # Init Params self.debug = debug self.tls_verify = tls_verify self.strict_errors = strict_errors self.tls_warnings = tls_warnings self.client_endpoint = client_endpoint self.cdp_credentials = cdp_credentials self.scrub_inputs = scrub_inputs self.cp_region = cp_region self.agent_header = agent_header if agent_header is not None else 'CDPY' # Setup self.throw_error = error_handler if error_handler else self._default_throw_error self.throw_warning = warning_handler if warning_handler else self._default_throw_warning self._clients = {} self.DEFAULT_PAGE_SIZE = 100 _loader = Loader() _user_agent = self._make_user_agent_header() self._client_creator = ClientCreator( _loader, Context(), EndpointCreator(EndpointResolver()), _user_agent, ResponseParserFactory(), create_retry_handler(self._load_retry_config(_loader))) # Logging _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' if debug: self._setup_logger(logging.DEBUG, _log_format) self.logger.debug("CDP SDK version: %s", _user_agent) else: self._setup_logger(logging.ERROR, _log_format) if self.tls_warnings is False: urllib3.disable_warnings(InsecureRequestWarning) # Warnings def _warning_format(message, category, filename, lineno, line=None): return ' %s:%s: %s:%s' % (filename, lineno, category.__name__, message) warnings.formatwarning = _warning_format # State listings # https://github.com/hortonworks/cloudbreak/blob/master/cluster-api/src/main/java/com/sequenceiq/ # cloudbreak/cluster/status/ClusterStatus.java#L8-L18 # https://github.com/hortonworks/cloudbreak/blob/master/core-api/src/main/java/com/sequenceiq/ # cloudbreak/api/endpoint/v4/common/Status.java#L14-L53 self.CREATION_STATES = [ 'REQUESTED', 'EXTERNAL_DATABASE_CREATION_IN_PROGRESS', 'STACK_CREATION_IN_PROGRESS', 'CREATION_INITIATED', 'FREEIPA_CREATION_IN_PROGRESS', 'STARTING', 'ENABLING', # DF 'provision:started', # ML 'installation:started' # ML ] self.TERMINATION_STATES = [ 'EXTERNAL_DATABASE_DELETION_IN_PROGRESS', 'STACK_DELETION_IN_PROGRESS', 'FREEIPA_DELETE_IN_PROGRESS', 'STOPPING', 'deprovision:started', # ML 'DISABLING' # DF ] self.STARTED_STATES = [ 'EXTERNAL_DATABASE_START_IN_PROGRESS', 'AVAILABLE', 'START_IN_PROGRESS', 'RUNNING', 'installation:finished', # ML 'Running', # DW 'GOOD_HEALTH', # DF 'ClusterCreationCompleted' #DE ] self.STOPPED_STATES = [ 'EXTERNAL_DATABASE_STOP_IN_PROGRESS', 'STOP_IN_PROGRESS', 'STOPPED', 'ENV_STOPPED', 'Stopped', # DW 'NOT_ENABLED', # DF 'ClusterDeletionCompleted', 'AppDeleted' # DE ] self.FAILED_STATES = [ 'PROVISIONING_FAILED', 'CREATE_FAILED', 'REJECTED', 'FAILED', 'TIMEDOUT', 'DELETE_FAILED', 'Error', # DW 'installation:failed', # ML 'provision:failed', # ML 'deprovision:failed', # ML 'BAD_HEALTH', # DF # DE service (all intermediate failure states, until CDE exposes a higher-level summary state) 'ClusterChartInstallationFailed', 'ClusterDNSCreationFailed', 'ClusterDNSDeletionFailed', 'ClusterIngressCreationFailed', 'ClusterProvisioningFailed', 'DBProvisioningFailed', 'FSMountTargetsCreationFailed', 'FSProvisioningFailed', 'ClusterTLSCertCreationFailed', 'ClusterServiceMeshProvisioningFailed', 'ClusterMonitoringConfigurationFailed', 'ClusterChartDeletionFailed', 'ClusterDeletionFailed', 'ClusterNamespaceDeletionFailed', 'DBDeletionFailed', 'FSMountTargetsDeletionFailed', 'FSDeletionFailed', 'ClusterTLSCertDeletionFailed', 'ClusterServiceMeshDeletionFailed', 'ClusterAccessGroupCreationFailed', 'ClusterAccessGroupDeletionFailed', 'ClusterUserSyncCheckFailed', 'ClusterCreationFailed', 'ClusterDeleteFromDBFailed', 'ClusterMaintenanceFailed', 'ClusterTLSCertRenewalFailed', # DE virtual cluster 'AppInstallationFailed', 'AppDeletionFailed' ] self.REMOVABLE_STATES = [ 'AVAILABLE', 'UPDATE_FAILED', 'CREATE_FAILED', 'ENABLE_SECURITY_FAILED', 'DELETE_FAILED', 'DELETE_COMPLETED', 'DELETED_ON_PROVIDER_SIDE', 'STOPPED', 'START_FAILED', 'STOP_FAILED', 'installation:failed', 'deprovision:failed', 'installation:finished', 'modify:finished', # ML 'Error', 'Running', 'Stopped', 'Deleting', # DW 'GOOD_HEALTH', 'CONCERNING_HEALTH', 'BAD_HEALTH', # DF 'ClusterCreationCompleted', 'AppInstalled', 'ClusterProvisioningFailed' #DE ] # common regex patterns self.DATAHUB_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.DATALAKE_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.ENV_NAME_PATTERN = re.compile(r'(^[^a-z0-9]|[^a-z0-9-]|^.{,4}$|^.{29,}$)') self.CREDENTIAL_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.OPERATION_REGEX = re.compile(r'operation ([0-9a-zA-Z-]{36}) running') # Workload services with special credential and endpoint handling self.WORKLOAD_SERVICES = ['dfworkload'] # substrings to check for in different CRNs self.CRN_STRINGS = { 'generic': ['crn:'], 'env': [':environments:', ':environment:'], 'df': [':df:', ':service:'], 'flow': [':df:', ':flow:'], 'readyflow': [':df:', 'readyFlow'], 'deployment': [':df:', ':deployment:'] }
def __init__(self, debug=False, tls_verify=False, strict_errors=False, tls_warnings=False, client_endpoint=None, cdp_credentials=None, error_handler=None, warning_handler=None, scrub_inputs=True): # Init Params self.debug = debug self.tls_verify = tls_verify self.strict_errors = strict_errors self.tls_warnings = tls_warnings self.client_endpoint = client_endpoint self.cdp_credentials = cdp_credentials self.scrub_inputs = scrub_inputs # Setup self.throw_error = error_handler if error_handler else self._default_throw_error self.throw_warning = warning_handler if warning_handler else self._default_throw_warning self._clients = {} self._PAGE_SIZE = 50 _loader = Loader() _user_agent = self._make_user_agent_header() self._client_creator = ClientCreator( _loader, Context(), EndpointCreator(EndpointResolver()), _user_agent, ResponseParserFactory(), create_retry_handler(self._load_retry_config(_loader))) # Logging _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' if debug: self._setup_logger(logging.DEBUG, _log_format) self.logger.debug("CDP SDK version: %s", _user_agent) else: self._setup_logger(logging.ERROR, _log_format) if self.tls_warnings is False: urllib3.disable_warnings(InsecureRequestWarning) # Warnings def _warning_format(message, category, filename, lineno, line=None): return ' %s:%s: %s:%s' % (filename, lineno, category.__name__, message) warnings.formatwarning = _warning_format # State listings # https://github.com/hortonworks/cloudbreak/blob/master/cluster-api/src/main/java/com/sequenceiq/ # cloudbreak/cluster/status/ClusterStatus.java#L8-L18 # https://github.com/hortonworks/cloudbreak/blob/master/core-api/src/main/java/com/sequenceiq/ # cloudbreak/api/endpoint/v4/common/Status.java#L14-L53 self.CREATION_STATES = [ 'REQUESTED', 'EXTERNAL_DATABASE_CREATION_IN_PROGRESS', 'STACK_CREATION_IN_PROGRESS', 'CREATION_INITIATED', 'FREEIPA_CREATION_IN_PROGRESS', 'STARTING' ] self.TERMINATION_STATES = [ 'EXTERNAL_DATABASE_DELETION_IN_PROGRESS', 'STACK_DELETION_IN_PROGRESS', 'FREEIPA_DELETE_IN_PROGRESS', 'STOPPING' ] self.STARTED_STATES = [ 'EXTERNAL_DATABASE_START_IN_PROGRESS', 'AVAILABLE', 'START_IN_PROGRESS', 'RUNNING', 'Running' ] self.STOPPED_STATES = [ 'EXTERNAL_DATABASE_STOP_IN_PROGRESS', 'STOP_IN_PROGRESS', 'STOPPED', 'ENV_STOPPED' ] self.FAILED_STATES = [ 'PROVISIONING_FAILED', 'CREATE_FAILED', 'REJECTED', 'FAILED', 'TIMEDOUT', 'DELETE_FAILED', 'Error' ] self.REMOVABLE_STATES = [ 'AVAILABLE', 'UPDATE_FAILED', 'CREATE_FAILED', 'ENABLE_SECURITY_FAILED', 'DELETE_FAILED', 'DELETE_COMPLETED', 'DELETED_ON_PROVIDER_SIDE', 'STOPPED', 'START_FAILED', 'STOP_FAILED', 'Error', 'Running' ] # common regex patterns self.DATAHUB_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.DATALAKE_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.ENV_NAME_PATTERN = re.compile( r'(^[^a-z0-9]|[^a-z0-9-]|^.{,4}$|^.{29,}$)') self.CREDENTIAL_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.OPERATION_REGEX = re.compile( r'operation ([0-9a-zA-Z-]{36}) running')