def setUp(self):
self.service_model = Mock(endpoint_name='iam',
signature_version='v1',
signing_name='iam')
self.cdp_service_model = Mock(endpoint_name='datahub',
endpoint_prefix='api',
signature_version='v1',
signing_name='datahub',
products=['CDP'])
self.environ = {}
self.environ_patch = patch('os.environ', self.environ)
self.environ_patch.start()
self.creator = EndpointCreator(EndpointResolver())
self.factory_patch = patch('cdpcli.parser.ResponseParserFactory')
self.factory = self.factory_patch.start()
self.retry_handler = Mock()
def __init__(self):
self._loader = Loader()
self._endpoint_creator = EndpointCreator(EndpointResolver())
self._user_agent_header = self._build_user_agent_header()
self._response_parser_factory = ResponseParserFactory()
self._cli_data = self._loader.load_json('cli.json')
self._retryhandler = self._create_default_retryhandler()
self._available_services = self._loader.list_available_services()
self._command_table = self._build_command_table()
self._argument_table = self._build_argument_table()
self._client_creator = ClientCreator(self._loader, Context(),
self._endpoint_creator,
self._user_agent_header,
self._response_parser_factory,
self._retryhandler)
def __init__(self, debug=False, tls_verify=False, strict_errors=False, tls_warnings=False, client_endpoint=None,
cdp_credentials=None, error_handler=None, warning_handler=None, scrub_inputs=True, cp_region='default',
agent_header=None):
# Init Params
self.debug = debug
self.tls_verify = tls_verify
self.strict_errors = strict_errors
self.tls_warnings = tls_warnings
self.client_endpoint = client_endpoint
self.cdp_credentials = cdp_credentials
self.scrub_inputs = scrub_inputs
self.cp_region = cp_region
self.agent_header = agent_header if agent_header is not None else 'CDPY'
# Setup
self.throw_error = error_handler if error_handler else self._default_throw_error
self.throw_warning = warning_handler if warning_handler else self._default_throw_warning
self._clients = {}
self.DEFAULT_PAGE_SIZE = 100
_loader = Loader()
_user_agent = self._make_user_agent_header()
self._client_creator = ClientCreator(
_loader,
Context(),
EndpointCreator(EndpointResolver()),
_user_agent,
ResponseParserFactory(),
create_retry_handler(self._load_retry_config(_loader)))
# Logging
_log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s'
if debug:
self._setup_logger(logging.DEBUG, _log_format)
self.logger.debug("CDP SDK version: %s", _user_agent)
else:
self._setup_logger(logging.ERROR, _log_format)
if self.tls_warnings is False:
urllib3.disable_warnings(InsecureRequestWarning)
# Warnings
def _warning_format(message, category, filename, lineno, line=None):
return ' %s:%s: %s:%s' % (filename, lineno, category.__name__, message)
warnings.formatwarning = _warning_format
# State listings
# https://github.com/hortonworks/cloudbreak/blob/master/cluster-api/src/main/java/com/sequenceiq/
# cloudbreak/cluster/status/ClusterStatus.java#L8-L18
# https://github.com/hortonworks/cloudbreak/blob/master/core-api/src/main/java/com/sequenceiq/
# cloudbreak/api/endpoint/v4/common/Status.java#L14-L53
self.CREATION_STATES = [
'REQUESTED',
'EXTERNAL_DATABASE_CREATION_IN_PROGRESS',
'STACK_CREATION_IN_PROGRESS',
'CREATION_INITIATED',
'FREEIPA_CREATION_IN_PROGRESS',
'STARTING',
'ENABLING', # DF
'provision:started', # ML
'installation:started' # ML
]
self.TERMINATION_STATES = [
'EXTERNAL_DATABASE_DELETION_IN_PROGRESS',
'STACK_DELETION_IN_PROGRESS',
'FREEIPA_DELETE_IN_PROGRESS',
'STOPPING',
'deprovision:started', # ML
'DISABLING' # DF
]
self.STARTED_STATES = [
'EXTERNAL_DATABASE_START_IN_PROGRESS',
'AVAILABLE',
'START_IN_PROGRESS',
'RUNNING',
'installation:finished', # ML
'Running', # DW
'GOOD_HEALTH', # DF
'ClusterCreationCompleted' #DE
]
self.STOPPED_STATES = [
'EXTERNAL_DATABASE_STOP_IN_PROGRESS',
'STOP_IN_PROGRESS',
'STOPPED',
'ENV_STOPPED',
'Stopped', # DW
'NOT_ENABLED', # DF
'ClusterDeletionCompleted', 'AppDeleted' # DE
]
self.FAILED_STATES = [
'PROVISIONING_FAILED',
'CREATE_FAILED',
'REJECTED',
'FAILED',
'TIMEDOUT',
'DELETE_FAILED',
'Error', # DW
'installation:failed', # ML
'provision:failed', # ML
'deprovision:failed', # ML
'BAD_HEALTH', # DF
# DE service (all intermediate failure states, until CDE exposes a higher-level summary state)
'ClusterChartInstallationFailed', 'ClusterDNSCreationFailed', 'ClusterDNSDeletionFailed',
'ClusterIngressCreationFailed', 'ClusterProvisioningFailed', 'DBProvisioningFailed',
'FSMountTargetsCreationFailed', 'FSProvisioningFailed', 'ClusterTLSCertCreationFailed',
'ClusterServiceMeshProvisioningFailed', 'ClusterMonitoringConfigurationFailed',
'ClusterChartDeletionFailed', 'ClusterDeletionFailed', 'ClusterNamespaceDeletionFailed',
'DBDeletionFailed', 'FSMountTargetsDeletionFailed', 'FSDeletionFailed',
'ClusterTLSCertDeletionFailed', 'ClusterServiceMeshDeletionFailed',
'ClusterAccessGroupCreationFailed', 'ClusterAccessGroupDeletionFailed',
'ClusterUserSyncCheckFailed', 'ClusterCreationFailed', 'ClusterDeleteFromDBFailed',
'ClusterMaintenanceFailed', 'ClusterTLSCertRenewalFailed',
# DE virtual cluster
'AppInstallationFailed', 'AppDeletionFailed'
]
self.REMOVABLE_STATES = [
'AVAILABLE', 'UPDATE_FAILED', 'CREATE_FAILED', 'ENABLE_SECURITY_FAILED', 'DELETE_FAILED',
'DELETE_COMPLETED', 'DELETED_ON_PROVIDER_SIDE', 'STOPPED', 'START_FAILED', 'STOP_FAILED',
'installation:failed', 'deprovision:failed', 'installation:finished', 'modify:finished', # ML
'Error', 'Running', 'Stopped', 'Deleting', # DW
'GOOD_HEALTH', 'CONCERNING_HEALTH', 'BAD_HEALTH', # DF
'ClusterCreationCompleted', 'AppInstalled', 'ClusterProvisioningFailed' #DE
]
# common regex patterns
self.DATAHUB_NAME_PATTERN = re.compile(r'[^a-z0-9-]')
self.DATALAKE_NAME_PATTERN = re.compile(r'[^a-z0-9-]')
self.ENV_NAME_PATTERN = re.compile(r'(^[^a-z0-9]|[^a-z0-9-]|^.{,4}$|^.{29,}$)')
self.CREDENTIAL_NAME_PATTERN = re.compile(r'[^a-z0-9-]')
self.OPERATION_REGEX = re.compile(r'operation ([0-9a-zA-Z-]{36}) running')
# Workload services with special credential and endpoint handling
self.WORKLOAD_SERVICES = ['dfworkload']
# substrings to check for in different CRNs
self.CRN_STRINGS = {
'generic': ['crn:'],
'env': [':environments:', ':environment:'],
'df': [':df:', ':service:'],
'flow': [':df:', ':flow:'],
'readyflow': [':df:', 'readyFlow'],
'deployment': [':df:', ':deployment:']
}
class TestEndpointCreator(unittest.TestCase):
def setUp(self):
self.service_model = Mock(endpoint_name='iam',
signature_version='v1',
signing_name='iam')
self.cdp_service_model = Mock(endpoint_name='datahub',
endpoint_prefix='api',
signature_version='v1',
signing_name='datahub',
products=['CDP'])
self.environ = {}
self.environ_patch = patch('os.environ', self.environ)
self.environ_patch.start()
self.creator = EndpointCreator(EndpointResolver())
self.factory_patch = patch('cdpcli.parser.ResponseParserFactory')
self.factory = self.factory_patch.start()
self.retry_handler = Mock()
def tearDown(self):
self.environ_patch.stop()
def test_altus_endpoint(self):
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url=None,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host,
'https://iamapi.us-west-1.altus.cloudera.com:443')
def test_cdp_endpoint(self):
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=None,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host,
'https://api.us-west-1.cdp.cloudera.com:443')
def test_cdp_endpoint_with_prefix(self):
self.cdp_service_model.endpoint_prefix = 'prefix'
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=None,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host,
'https://prefix.us-west-1.cdp.cloudera.com:443')
def test_creates_endpoint_with_configured_url(self):
endpoint_url = 'https://endpoint.url'
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url=endpoint_url,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_endpoint_with_customized_timeout(self):
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url='https://example.com',
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.timeout, 123)
def test_create_endpoint_with_config_url_altus(self):
endpoint_url = 'https://endpoint-config.url'
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url=None,
scoped_config={
"endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_endpoint_with_config_url_cdp(self):
endpoint_url = 'https://endpoint-config.url'
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=None,
scoped_config={
"cdp_endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_endpoint_with_configured_url_substitution(self):
endpoint_url = 'https://%s.endpoint.url'
expected = endpoint_url % self.service_model.endpoint_name
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url=endpoint_url,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, expected)
def test_create_endpoint_with_configured_cdp_url_substitution(self):
endpoint_url = 'https://%s.endpoint.url'
self.cdp_service_model.endpoint_prefix = 'prefix'
expected = endpoint_url % self.cdp_service_model.endpoint_prefix
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=endpoint_url,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, expected)
def test_create_endpoint_with_config_url_substitution(self):
endpoint_url = 'https://%s.endpoint-config.url'
expected = endpoint_url % self.service_model.endpoint_name
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url=None,
scoped_config={
"endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, expected)
def test_create_endpoint_with_config_cdp_url_substitution(self):
endpoint_url = 'https://%s.endpoint-config.url'
self.cdp_service_model.endpoint_prefix = 'prefix'
expected = endpoint_url % self.cdp_service_model.endpoint_prefix
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=None,
scoped_config={
"cdp_endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, expected)
def test_create_endpoint_with_configured_url_bad_substitution(self):
endpoint_url = 'https://%s.%s.endpoint.url'
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url=endpoint_url,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_endpoint_with_configured_cdp_url_bad_substitution(self):
endpoint_url = 'https://%s.%s.endpoint.url'
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=endpoint_url,
scoped_config={},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_endpoint_with_config_url_bad_substitution(self):
endpoint_url = 'https://%s.%s.endpoint-config.url'
endpoint = self.creator.create_endpoint(
self.service_model,
endpoint_url=None,
scoped_config={
"endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_endpoint_with_config_cdp_url_bad_substitution(self):
endpoint_url = 'https://%s.%s.endpoint-config.url'
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=None,
scoped_config={
"cdp_endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_endpoint_for_internal_apis(self):
endpoint_url = 'https://foo-api.internal.url'
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=None,
scoped_config={
"cdp_endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, endpoint_url)
def test_create_wildcard_endpoint_for_internal_apis(self):
endpoint_url = 'https://%s-api.internal.url'
self.cdp_service_model.endpoint_prefix = 'prefix'
expected = endpoint_url % self.cdp_service_model.endpoint_prefix
endpoint = self.creator.create_endpoint(
self.cdp_service_model,
endpoint_url=None,
scoped_config={
"cdp_endpoint_url": endpoint_url,
"config1": "value1"
},
timeout=123,
response_parser_factory=self.factory_patch,
tls_verification=False,
retry_handler=self.retry_handler)
self.assertEqual(endpoint.host, expected)
def __init__(self,
debug=False,
tls_verify=False,
strict_errors=False,
tls_warnings=False,
client_endpoint=None,
cdp_credentials=None,
error_handler=None,
warning_handler=None,
scrub_inputs=True):
# Init Params
self.debug = debug
self.tls_verify = tls_verify
self.strict_errors = strict_errors
self.tls_warnings = tls_warnings
self.client_endpoint = client_endpoint
self.cdp_credentials = cdp_credentials
self.scrub_inputs = scrub_inputs
# Setup
self.throw_error = error_handler if error_handler else self._default_throw_error
self.throw_warning = warning_handler if warning_handler else self._default_throw_warning
self._clients = {}
self._PAGE_SIZE = 50
_loader = Loader()
_user_agent = self._make_user_agent_header()
self._client_creator = ClientCreator(
_loader, Context(), EndpointCreator(EndpointResolver()),
_user_agent, ResponseParserFactory(),
create_retry_handler(self._load_retry_config(_loader)))
# Logging
_log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s'
if debug:
self._setup_logger(logging.DEBUG, _log_format)
self.logger.debug("CDP SDK version: %s", _user_agent)
else:
self._setup_logger(logging.ERROR, _log_format)
if self.tls_warnings is False:
urllib3.disable_warnings(InsecureRequestWarning)
# Warnings
def _warning_format(message, category, filename, lineno, line=None):
return ' %s:%s: %s:%s' % (filename, lineno, category.__name__,
message)
warnings.formatwarning = _warning_format
# State listings
# https://github.com/hortonworks/cloudbreak/blob/master/cluster-api/src/main/java/com/sequenceiq/
# cloudbreak/cluster/status/ClusterStatus.java#L8-L18
# https://github.com/hortonworks/cloudbreak/blob/master/core-api/src/main/java/com/sequenceiq/
# cloudbreak/api/endpoint/v4/common/Status.java#L14-L53
self.CREATION_STATES = [
'REQUESTED', 'EXTERNAL_DATABASE_CREATION_IN_PROGRESS',
'STACK_CREATION_IN_PROGRESS', 'CREATION_INITIATED',
'FREEIPA_CREATION_IN_PROGRESS', 'STARTING'
]
self.TERMINATION_STATES = [
'EXTERNAL_DATABASE_DELETION_IN_PROGRESS',
'STACK_DELETION_IN_PROGRESS', 'FREEIPA_DELETE_IN_PROGRESS',
'STOPPING'
]
self.STARTED_STATES = [
'EXTERNAL_DATABASE_START_IN_PROGRESS', 'AVAILABLE',
'START_IN_PROGRESS', 'RUNNING', 'Running'
]
self.STOPPED_STATES = [
'EXTERNAL_DATABASE_STOP_IN_PROGRESS', 'STOP_IN_PROGRESS',
'STOPPED', 'ENV_STOPPED'
]
self.FAILED_STATES = [
'PROVISIONING_FAILED', 'CREATE_FAILED', 'REJECTED', 'FAILED',
'TIMEDOUT', 'DELETE_FAILED', 'Error'
]
self.REMOVABLE_STATES = [
'AVAILABLE', 'UPDATE_FAILED', 'CREATE_FAILED',
'ENABLE_SECURITY_FAILED', 'DELETE_FAILED', 'DELETE_COMPLETED',
'DELETED_ON_PROVIDER_SIDE', 'STOPPED', 'START_FAILED',
'STOP_FAILED', 'Error', 'Running'
]
# common regex patterns
self.DATAHUB_NAME_PATTERN = re.compile(r'[^a-z0-9-]')
self.DATALAKE_NAME_PATTERN = re.compile(r'[^a-z0-9-]')
self.ENV_NAME_PATTERN = re.compile(
r'(^[^a-z0-9]|[^a-z0-9-]|^.{,4}$|^.{29,}$)')
self.CREDENTIAL_NAME_PATTERN = re.compile(r'[^a-z0-9-]')
self.OPERATION_REGEX = re.compile(
r'operation ([0-9a-zA-Z-]{36}) running')