def __init__(self): self._loader = Loader() self._endpoint_creator = EndpointCreator(EndpointResolver()) self._user_agent_header = self._build_user_agent_header() self._response_parser_factory = ResponseParserFactory() self._cli_data = self._loader.load_json('cli.json') self._retryhandler = self._create_default_retryhandler() self._available_services = self._loader.list_available_services() self._command_table = self._build_command_table() self._argument_table = self._build_argument_table() self._client_creator = ClientCreator(self._loader, Context(), self._endpoint_creator, self._user_agent_header, self._response_parser_factory, self._retryhandler)
def __init__(self, host, proxies=None, tls_verification=True, timeout=DEFAULT_TIMEOUT, response_parser_factory=None, retry_handler=None): self.host = host self.tls_verification = tls_verification if proxies is None: proxies = {} self.proxies = proxies self.http_session = PreserveAuthSession() self.timeout = timeout self._lock = threading.Lock() if response_parser_factory is None: response_parser_factory = ResponseParserFactory() self._response_parser_factory = response_parser_factory self._retry_handler = retry_handler
def setUp(self): self.loader = Mock() self.loader.load_service_data.return_value = yaml.safe_load( open(os.path.join(CLIENT_DIR, 'service.yaml'))) self.endpoint = Mock() self.endpoint.host = 'http://thunderhead.cloudera.altus.cloudera.com' self.endpoint.make_request.return_value = (Mock(status_code=200), {}) self.endpoint_creator = Mock() self.endpoint_creator.create_endpoint.return_value = self.endpoint self.retryhandler = Mock() self.context = FakeContext({}) self.client_creator = ClientCreator(self.loader, self.context, self.endpoint_creator, 'user-agent-header', ResponseParserFactory(), self.retryhandler) self.credentials = Mock() self.credentials.private_key = "A Private Key"
def setUp(self): self.model = yaml.safe_load( open(os.path.join(MODEL_DIR, 'service.yaml'))) self.service_model = ServiceModel(self.model, 'servicename') self.loader = Mock() self.loader.load_service_data.return_value = self.model self.endpoint = Mock() self.endpoint.host = 'http://thunderhead.cloudera.altus.cloudera.com' self.endpoint.make_request.return_value = (Mock(status_code=200), {}) self.endpoint_creator = Mock() self.endpoint_creator.create_endpoint.return_value = self.endpoint self.context = Mock() self.context.get_credentials.return_value = Mock( private_key="A private key") self.context.get_scoped_config.return_value = {} self.client_creator = ClientCreator(self.loader, self.context, self.endpoint_creator, 'user-agent-header', ResponseParserFactory(), Mock()) self.parsed_globals = Mock(connect_timeout=1, read_timeout=1)
def __init__(self, debug=False, tls_verify=False, strict_errors=False, tls_warnings=False, client_endpoint=None, cdp_credentials=None, error_handler=None, warning_handler=None, scrub_inputs=True, cp_region='default', agent_header=None): # Init Params self.debug = debug self.tls_verify = tls_verify self.strict_errors = strict_errors self.tls_warnings = tls_warnings self.client_endpoint = client_endpoint self.cdp_credentials = cdp_credentials self.scrub_inputs = scrub_inputs self.cp_region = cp_region self.agent_header = agent_header if agent_header is not None else 'CDPY' # Setup self.throw_error = error_handler if error_handler else self._default_throw_error self.throw_warning = warning_handler if warning_handler else self._default_throw_warning self._clients = {} self.DEFAULT_PAGE_SIZE = 100 _loader = Loader() _user_agent = self._make_user_agent_header() self._client_creator = ClientCreator( _loader, Context(), EndpointCreator(EndpointResolver()), _user_agent, ResponseParserFactory(), create_retry_handler(self._load_retry_config(_loader))) # Logging _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' if debug: self._setup_logger(logging.DEBUG, _log_format) self.logger.debug("CDP SDK version: %s", _user_agent) else: self._setup_logger(logging.ERROR, _log_format) if self.tls_warnings is False: urllib3.disable_warnings(InsecureRequestWarning) # Warnings def _warning_format(message, category, filename, lineno, line=None): return ' %s:%s: %s:%s' % (filename, lineno, category.__name__, message) warnings.formatwarning = _warning_format # State listings # https://github.com/hortonworks/cloudbreak/blob/master/cluster-api/src/main/java/com/sequenceiq/ # cloudbreak/cluster/status/ClusterStatus.java#L8-L18 # https://github.com/hortonworks/cloudbreak/blob/master/core-api/src/main/java/com/sequenceiq/ # cloudbreak/api/endpoint/v4/common/Status.java#L14-L53 self.CREATION_STATES = [ 'REQUESTED', 'EXTERNAL_DATABASE_CREATION_IN_PROGRESS', 'STACK_CREATION_IN_PROGRESS', 'CREATION_INITIATED', 'FREEIPA_CREATION_IN_PROGRESS', 'STARTING', 'ENABLING', # DF 'provision:started', # ML 'installation:started' # ML ] self.TERMINATION_STATES = [ 'EXTERNAL_DATABASE_DELETION_IN_PROGRESS', 'STACK_DELETION_IN_PROGRESS', 'FREEIPA_DELETE_IN_PROGRESS', 'STOPPING', 'deprovision:started', # ML 'DISABLING' # DF ] self.STARTED_STATES = [ 'EXTERNAL_DATABASE_START_IN_PROGRESS', 'AVAILABLE', 'START_IN_PROGRESS', 'RUNNING', 'installation:finished', # ML 'Running', # DW 'GOOD_HEALTH', # DF 'ClusterCreationCompleted' #DE ] self.STOPPED_STATES = [ 'EXTERNAL_DATABASE_STOP_IN_PROGRESS', 'STOP_IN_PROGRESS', 'STOPPED', 'ENV_STOPPED', 'Stopped', # DW 'NOT_ENABLED', # DF 'ClusterDeletionCompleted', 'AppDeleted' # DE ] self.FAILED_STATES = [ 'PROVISIONING_FAILED', 'CREATE_FAILED', 'REJECTED', 'FAILED', 'TIMEDOUT', 'DELETE_FAILED', 'Error', # DW 'installation:failed', # ML 'provision:failed', # ML 'deprovision:failed', # ML 'BAD_HEALTH', # DF # DE service (all intermediate failure states, until CDE exposes a higher-level summary state) 'ClusterChartInstallationFailed', 'ClusterDNSCreationFailed', 'ClusterDNSDeletionFailed', 'ClusterIngressCreationFailed', 'ClusterProvisioningFailed', 'DBProvisioningFailed', 'FSMountTargetsCreationFailed', 'FSProvisioningFailed', 'ClusterTLSCertCreationFailed', 'ClusterServiceMeshProvisioningFailed', 'ClusterMonitoringConfigurationFailed', 'ClusterChartDeletionFailed', 'ClusterDeletionFailed', 'ClusterNamespaceDeletionFailed', 'DBDeletionFailed', 'FSMountTargetsDeletionFailed', 'FSDeletionFailed', 'ClusterTLSCertDeletionFailed', 'ClusterServiceMeshDeletionFailed', 'ClusterAccessGroupCreationFailed', 'ClusterAccessGroupDeletionFailed', 'ClusterUserSyncCheckFailed', 'ClusterCreationFailed', 'ClusterDeleteFromDBFailed', 'ClusterMaintenanceFailed', 'ClusterTLSCertRenewalFailed', # DE virtual cluster 'AppInstallationFailed', 'AppDeletionFailed' ] self.REMOVABLE_STATES = [ 'AVAILABLE', 'UPDATE_FAILED', 'CREATE_FAILED', 'ENABLE_SECURITY_FAILED', 'DELETE_FAILED', 'DELETE_COMPLETED', 'DELETED_ON_PROVIDER_SIDE', 'STOPPED', 'START_FAILED', 'STOP_FAILED', 'installation:failed', 'deprovision:failed', 'installation:finished', 'modify:finished', # ML 'Error', 'Running', 'Stopped', 'Deleting', # DW 'GOOD_HEALTH', 'CONCERNING_HEALTH', 'BAD_HEALTH', # DF 'ClusterCreationCompleted', 'AppInstalled', 'ClusterProvisioningFailed' #DE ] # common regex patterns self.DATAHUB_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.DATALAKE_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.ENV_NAME_PATTERN = re.compile(r'(^[^a-z0-9]|[^a-z0-9-]|^.{,4}$|^.{29,}$)') self.CREDENTIAL_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.OPERATION_REGEX = re.compile(r'operation ([0-9a-zA-Z-]{36}) running') # Workload services with special credential and endpoint handling self.WORKLOAD_SERVICES = ['dfworkload'] # substrings to check for in different CRNs self.CRN_STRINGS = { 'generic': ['crn:'], 'env': [':environments:', ':environment:'], 'df': [':df:', ':service:'], 'flow': [':df:', ':flow:'], 'readyflow': [':df:', 'readyFlow'], 'deployment': [':df:', ':deployment:'] }
def setUp(self): self.model = yaml.safe_load( open(os.path.join(PROTOCOL_DIR, 'service.yaml'))) self.service_model = ServiceModel(self.model, 'servicename') self.parser = ResponseParserFactory().create_parser()
class TestParser(unittest.TestCase): def setUp(self): self.model = yaml.safe_load( open(os.path.join(PROTOCOL_DIR, 'service.yaml'))) self.service_model = ServiceModel(self.model, 'servicename') self.parser = ResponseParserFactory().create_parser() def _create_response(self, body): response = {} response["status_code"] = 200 response["headers"] = {} response["body"] = json.dumps(body).encode(UTF8) return response def _create_error_response(self, code, message): body = {} body['code'] = code body['message'] = message response = {} response["status_code"] = 301 response["body"] = json.dumps(body).encode(UTF8) return response def _assert_same(self, v1, v2, path=""): if type(v1) is dict: for k in v1.keys(): self.assertTrue(k in v2) self._assert_same(v1[k], v2[k]) for k in v2.keys(): self.assertTrue(k in v1) self._assert_same(v1[k], v2[k]) return if type(v1) is datetime.datetime and isinstance(v2, six.text_type): v2 = dateutil.parser.parse(v2) if type(v2) is datetime.datetime and isinstance(v1, six.text_type): v1 = dateutil.parser.parse(v1) self.assertEquals(v1, v2) def _validate_parsed_response(self, response, parsed_response): self._assert_same(json.loads(response['body'].decode(UTF8)), parsed_response) def _validate_error_response(self, response, parsed_response): self._assert_same(json.loads(response['body'].decode(UTF8)), parsed_response['error']) def test_no_shape(self): director = {} director['name'] = 'test-name' director['crn'] = 'test-crn' director['endpoint'] = {'address': '1.2.3.4', 'port': 12345} response = self._create_response({'director': director}) parsed_response = self.parser.parse(response, None) self.assertEqual({}, parsed_response) def test_no_response(self): response = {} response["status_code"] = 200 response["headers"] = {} response["body"] = "" operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) self.assertEqual({}, parsed_response) response["body"] = None operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) self.assertEqual({}, parsed_response) response["body"] = {} operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) self.assertEqual({}, parsed_response) def test_non_json_response(self): response = {} response["status_code"] = 404 response["headers"] = {} response["body"] = "Page not found".encode(UTF8) operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) self.assertEqual( parsed_response, {'error': { 'code': 'UNKNOWN_ERROR', 'message': 'Page not found' }}) def test_response(self): director = {} director['name'] = 'test-name' director['crn'] = 'test-crn' director['endpoint'] = {'address': '1.2.3.4', 'port': 12345} director['creationDate'] = "2014-07-31T12:26:00.000-07:00" director['booleanData'] = True director['numberData'] = "123.456" response = self._create_response({'director': director}) operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) self._validate_parsed_response(response, parsed_response) def test_invalid_datetime(self): director = {} director['creationDate'] = "BAD!-2014-07-31T12:26:00.000-07:00-BAD!" response = self._create_response({'director': director}) operation_model = self.service_model.operation_model('createDirector') with self.assertRaises(ValueError): self.parser.parse(response, operation_model.output_shape) def test_response_with_array(self): director1 = {} director1['name'] = 'test-name-1' director1['crn'] = 'test-crn-1' director1['endpoint'] = {'address': '1.2.3.4', 'port': 12345} director2 = {} director2['name'] = 'test-name-2' director2['crn'] = 'test-crn-2' director2['endpoint'] = {'address': '2.3.4.5', 'port': 23456} response = self._create_response({'directors': [director1, director2]}) operation_model = self.service_model.operation_model( 'describeDirectors') parsed_response = self.parser.parse(response, operation_model.output_shape) self._validate_parsed_response(response, parsed_response) def test_response_with_missing_member(self): director = {} director['crn'] = 'test-crn' director['endpoint'] = {'address': '1.2.3.4', 'port': 12345} response = self._create_response({'director': director}) operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) self._validate_parsed_response(response, parsed_response) def test_response_with_none_member(self): director = {} director['name'] = None director['crn'] = 'test-crn' director['endpoint'] = {'address': '1.2.3.4', 'port': 12345} response = self._create_response({'director': director}) operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) del director['name'] response = self._create_response({'director': director}) self._validate_parsed_response(response, parsed_response) def test_error(self): response = self._create_error_response("YouMessedUpException", "You, sir, have messed up.") operation_model = self.service_model.operation_model('createDirector') parsed_response = self.parser.parse(response, operation_model.output_shape) self._validate_error_response(response, parsed_response)
def __init__(self, debug=False, tls_verify=False, strict_errors=False, tls_warnings=False, client_endpoint=None, cdp_credentials=None, error_handler=None, warning_handler=None, scrub_inputs=True): # Init Params self.debug = debug self.tls_verify = tls_verify self.strict_errors = strict_errors self.tls_warnings = tls_warnings self.client_endpoint = client_endpoint self.cdp_credentials = cdp_credentials self.scrub_inputs = scrub_inputs # Setup self.throw_error = error_handler if error_handler else self._default_throw_error self.throw_warning = warning_handler if warning_handler else self._default_throw_warning self._clients = {} self._PAGE_SIZE = 50 _loader = Loader() _user_agent = self._make_user_agent_header() self._client_creator = ClientCreator( _loader, Context(), EndpointCreator(EndpointResolver()), _user_agent, ResponseParserFactory(), create_retry_handler(self._load_retry_config(_loader))) # Logging _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' if debug: self._setup_logger(logging.DEBUG, _log_format) self.logger.debug("CDP SDK version: %s", _user_agent) else: self._setup_logger(logging.ERROR, _log_format) if self.tls_warnings is False: urllib3.disable_warnings(InsecureRequestWarning) # Warnings def _warning_format(message, category, filename, lineno, line=None): return ' %s:%s: %s:%s' % (filename, lineno, category.__name__, message) warnings.formatwarning = _warning_format # State listings # https://github.com/hortonworks/cloudbreak/blob/master/cluster-api/src/main/java/com/sequenceiq/ # cloudbreak/cluster/status/ClusterStatus.java#L8-L18 # https://github.com/hortonworks/cloudbreak/blob/master/core-api/src/main/java/com/sequenceiq/ # cloudbreak/api/endpoint/v4/common/Status.java#L14-L53 self.CREATION_STATES = [ 'REQUESTED', 'EXTERNAL_DATABASE_CREATION_IN_PROGRESS', 'STACK_CREATION_IN_PROGRESS', 'CREATION_INITIATED', 'FREEIPA_CREATION_IN_PROGRESS', 'STARTING' ] self.TERMINATION_STATES = [ 'EXTERNAL_DATABASE_DELETION_IN_PROGRESS', 'STACK_DELETION_IN_PROGRESS', 'FREEIPA_DELETE_IN_PROGRESS', 'STOPPING' ] self.STARTED_STATES = [ 'EXTERNAL_DATABASE_START_IN_PROGRESS', 'AVAILABLE', 'START_IN_PROGRESS', 'RUNNING', 'Running' ] self.STOPPED_STATES = [ 'EXTERNAL_DATABASE_STOP_IN_PROGRESS', 'STOP_IN_PROGRESS', 'STOPPED', 'ENV_STOPPED' ] self.FAILED_STATES = [ 'PROVISIONING_FAILED', 'CREATE_FAILED', 'REJECTED', 'FAILED', 'TIMEDOUT', 'DELETE_FAILED', 'Error' ] self.REMOVABLE_STATES = [ 'AVAILABLE', 'UPDATE_FAILED', 'CREATE_FAILED', 'ENABLE_SECURITY_FAILED', 'DELETE_FAILED', 'DELETE_COMPLETED', 'DELETED_ON_PROVIDER_SIDE', 'STOPPED', 'START_FAILED', 'STOP_FAILED', 'Error', 'Running' ] # common regex patterns self.DATAHUB_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.DATALAKE_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.ENV_NAME_PATTERN = re.compile( r'(^[^a-z0-9]|[^a-z0-9-]|^.{,4}$|^.{29,}$)') self.CREDENTIAL_NAME_PATTERN = re.compile(r'[^a-z0-9-]') self.OPERATION_REGEX = re.compile( r'operation ([0-9a-zA-Z-]{36}) running')