def get_kubeconf(self, client_config, params): """ get kubernetes configuration for cluster. """ cluster = \ self.client.describe_cluster(name=params.get(CLUSTER_NAME)) cluster_cert = cluster["cluster"]["certificateAuthority"]["data"] cluster_ep = cluster["cluster"]["endpoint"] sts_client = boto3.client('sts', **client_config) _register_cluster_name_handlers(sts_client) url = sts_client.generate_presigned_url( 'get_caller_identity', {'ClusterName': params.get(CLUSTER_NAME)}, HttpMethod='GET', ExpiresIn=TOKEN_EXPIRATION_MINS) encoded = base64.urlsafe_b64encode(url.encode('utf-8')) token = TOKEN_PREFIX + \ encoded.decode('utf-8').rstrip('=') # build the cluster config hash cluster_config = { "apiVersion": "v1", "kind": "Config", "clusters": [{ "cluster": { "server": text_type(cluster_ep), "certificate-authority-data": text_type(cluster_cert) }, "name": "kubernetes" }], "contexts": [{ "context": { "cluster": "kubernetes", "user": "******" }, "name": "aws" }], "current-context": "aws", "preferences": {}, "users": [{ "name": "aws", "user": { "token": token } }] } return cluster_config
def test_create_raises_UnknownServiceError(self): _test_name = 'test_create_UnknownServiceError' _test_node_properties = { 'use_external_resource': False, 'client_config': CLIENT_CONFIG } _test_runtime_properties = {'resource_config': {}} _ctx = self.get_mock_ctx( _test_name, test_properties=_test_node_properties, test_runtime_properties=_test_runtime_properties, type_hierarchy=OPTION_GROUP_TH) current_ctx.set(_ctx) with self.assertRaises(UnknownServiceError) as error: option_group.create(ctx=_ctx, resource_config=None, iface=None) self.assertEqual( text_type(error.exception), "Unknown service: 'rds'. Valid service names are: ['rds']") self.fake_boto.assert_called_with('rds', aws_access_key_id='xxx', aws_secret_access_key='yyy', region_name='aq-testzone-1')
def test_immortal_delete(self): _test_name = 'test_delete' _ctx = self.get_mock_ctx( _test_name, test_properties=NODE_PROPERTIES, test_runtime_properties=RUNTIME_PROPERTIES_AFTER_CREATE, type_hierarchy=OPTION_GROUP_TH) current_ctx.set(_ctx) self.fake_client.delete_option_group = MagicMock(return_value={}) self.fake_client.describe_option_groups = MagicMock( return_value={ 'OptionGroupsList': [{ 'OptionGroupName': 'dev-db-option-group', 'OptionGroupArn': 'OptionGroupArn' }] }) with self.assertRaises(OperationRetry) as error: option_group.delete(ctx=_ctx, resource_config=None, iface=None) self.assertEqual( text_type(error.exception), ('RDS Option Group ID# "dev-db-option-group" is still ' + 'in a pending state.'))
def create(ctx, iface, resource_config, **_): """Creates an AWS EKS Cluster""" store_kube_config_in_runtime = \ ctx.node.properties['store_kube_config_in_runtime'] params = dict() if not resource_config else resource_config.copy() resource_id = \ utils.get_resource_id( ctx.node, ctx.instance, params.get(CLUSTER_NAME), use_instance_id=True ) utils.update_resource_id(ctx.instance, resource_id) iface = prepare_describe_cluster_filter(resource_config.copy(), iface) response = iface.create(params) if response and response.get(CLUSTER): resource_arn = response.get(CLUSTER).get(CLUSTER_ARN) utils.update_resource_arn(ctx.instance, resource_arn) # wait for cluster to be active ctx.logger.info("Waiting for Cluster to become Active") iface.wait_for_cluster(params, 'cluster_active') if store_kube_config_in_runtime: try: client_config = ctx.node.properties['client_config'] kubeconf = iface.get_kubeconf(client_config, params) # check if kubeconf is json serializable or not json.dumps(kubeconf) ctx.instance.runtime_properties['kubeconf'] = kubeconf except TypeError as error: raise NonRecoverableError( 'kubeconf not json serializable {0}'.format(text_type(error)))
def _prepare_create_raises_UnknownServiceError( self, type_hierarchy, type_name, type_class, type_node='cloudify.nodes.Root'): _ctx = self.get_mock_ctx( 'test_create', test_properties=DEFAULT_NODE_PROPERTIES, test_runtime_properties=DEFAULT_RUNTIME_PROPERTIES, type_hierarchy=type_hierarchy, type_node=type_node) current_ctx.set(_ctx) fake_boto, fake_client = self.fake_boto_client(type_name) with patch('boto3.client', fake_boto): with self.assertRaises(UnknownServiceError) as error: type_class.create(ctx=_ctx, resource_config=None, iface=None) self.assertEqual(text_type(error.exception), ("Unknown service: '" + type_name + "'. Valid service names are: ['rds']")) fake_boto.assert_called_with(type_name, **CLIENT_CONFIG)
def _handle_userdata(existing_userdata): if existing_userdata is None: existing_userdata = '' elif isinstance(existing_userdata, dict) or \ isinstance(existing_userdata, list): existing_userdata = json.dumps(existing_userdata) elif not isinstance(existing_userdata, text_type): existing_userdata = text_type(existing_userdata) install_agent_userdata = ctx.agent.init_script() os_family = ctx.node.properties['os_family'] if not (existing_userdata or install_agent_userdata): return '' # Windows instances require no more than one # Powershell script, which must be surrounded by # Powershell tags. if install_agent_userdata and os_family == 'windows': # Get the powershell content from install_agent_userdata install_agent_userdata = \ extract_powershell_content(install_agent_userdata) # Get the powershell content from existing_userdata # (If it exists.) existing_userdata_powershell = \ extract_powershell_content(existing_userdata) # Combine the powershell content from two sources. install_agent_userdata = \ '#ps1_sysnative\n{0}\n{1}\n{2}\n{3}\n'.format( PS_OPEN, existing_userdata_powershell, install_agent_userdata, PS_CLOSE) # Additional work on the existing_userdata. # Remove duplicate Powershell content. # Get rid of unnecessary newlines. existing_userdata = \ existing_userdata.replace( existing_userdata_powershell, '').replace( PS_OPEN, '').replace( PS_CLOSE, '').strip() if not existing_userdata or existing_userdata.isspace(): final_userdata = install_agent_userdata elif not install_agent_userdata: final_userdata = existing_userdata else: final_userdata = compute.create_multi_mimetype_userdata( [existing_userdata, install_agent_userdata]) return final_userdata
def create(ctx, iface, resource_config, params, **_): """Creates an AWS Autoscaling Autoscaling Launch Configuration""" # Check if the "IamInstanceProfile" is passed or not and then update it iam_instance_profile = params.get(IAM_INSTANCE_PROFILE) if iam_instance_profile: if isinstance(iam_instance_profile, text_type): iam_instance_profile = iam_instance_profile.strip() params[IAM_INSTANCE_PROFILE] = text_type(iam_instance_profile) else: raise NonRecoverableError('Invalid {0} data type for {1}' ''.format(type(iam_instance_profile), IAM_INSTANCE_PROFILE)) # Add Security Groups secgroups_list = params.get(SECGROUPS, []) params[SECGROUPS] = \ utils.add_resources_from_rels( ctx.instance, SECGROUP_TYPE, secgroups_list) image_id = params.get(IMAGEID) # Add Instance and Instance Type instance_id = params.get(INSTANCEID) instance_type = params.get(INSTANCE_TYPE_PROPERTY) if not image_id and not instance_id: instance_id = utils.find_resource_id_by_type( ctx.instance, INSTANCE_TYPE_NEW) or \ utils.find_resource_id_by_type( ctx.instance, INSTANCE_TYPE) params.update({INSTANCEID: instance_id}) if instance_id and not instance_type: targ = utils.find_rel_by_node_type( ctx.instance, INSTANCE_TYPE_NEW) or \ utils.find_rel_by_node_type( ctx.instance, INSTANCE_TYPE) if targ: instance_type = \ targ.target.instance.runtime_properties.get( 'resource_config', {}).get( INSTANCE_TYPE_PROPERTY) or \ targ.target.node.properties.get( INSTANCE_TYPE_PROPERTY_DEPRECATED) params.update({INSTANCE_TYPE_PROPERTY: instance_type}) utils.update_resource_id(ctx.instance, params.get(RESOURCE_NAME)) iface.update_resource_id(params.get(RESOURCE_NAME)) # Actually create the resource if not iface.resource_id: setattr(iface, 'resource_id', params.get(RESOURCE_NAME)) iface.create(params) resource_arn = iface.properties[LC_ARN] utils.update_resource_arn(ctx.instance, resource_arn)
def create(ctx, iface, resource_config, **_): '''Creates AWS EC2 Keypairs''' params = \ dict() if not resource_config else resource_config.copy() params[KEYNAME] = utils.get_resource_name(params.get(KEYNAME)) key_name = params[KEYNAME] if PUBLIC_KEY_MATERIAL in params: create_response = \ iface.import_keypair( params, log_response=ctx.node.properties['log_create_response']) else: create_response = iface.create( params, log_response=ctx.node.properties['log_create_response']) # Allow the end user to store the key material in a secret. if ctx.node.properties['create_secret']: try: client = get_rest_client() except KeyError: # No pun intended. raise NonRecoverableError( 'create_secret is only supported with a Cloudify Manager.') # This makes the line too long for flake8 if included in args. secret_name = ctx.node.properties.get('secret_name', key_name) secrets_count = len(client.secrets.list(key=secret_name)) secret_value = create_response.get('KeyMaterial') try: if secrets_count == 0: client.secrets.create(key=secret_name, value=secret_value) elif secrets_count == 1 and \ ctx.node.properties.get( 'update_existing_secret', False) is True: client.secrets.update(key=secret_name, value=secret_value) except CloudifyClientError as e: raise NonRecoverableError(text_type(e)) cleaned_create_response = \ utils.JsonCleanuper(create_response).to_dict() # Allow the end user to opt-in to storing the key # material in the runtime properties. # Default is false if 'KeyMaterial' in cleaned_create_response and not \ ctx.node.properties['store_in_runtime_properties']: del cleaned_create_response['KeyMaterial'] ctx.instance.runtime_properties['create_response'] = \ cleaned_create_response iface.update_resource_id(cleaned_create_response.get(KEYNAME)) utils.update_resource_id(ctx.instance, key_name)
def prepare(ctx, resource_config, **_): '''Prepares an AWS Lambda Permission''' # Save the parameters if not utils.get_resource_id(): if resource_config.get('StatementId'): utils.update_resource_id(ctx.instance, resource_config['StatementId']) else: utils.update_resource_id(ctx.instance, text_type(uuid4())) ctx.instance.runtime_properties['resource_config'] = resource_config
def properties(self): '''Gets the properties of an external resource''' try: resources = self.client.describe_load_balancers( Names=[self.resource_id]) except (ClientError, ParamValidationError) as e: self.logger.warn('Ignoring error: {0}'.format(text_type(e))) else: if resources: return resources['LoadBalancers'][0] return {}
def _cleanuped_dict(self, resource): for k in resource: if not resource[k]: continue if isinstance(resource[k], list): self._cleanuped_list(resource[k]) elif isinstance(resource[k], dict): self._cleanuped_dict(resource[k]) elif (not isinstance(resource[k], int) and # integer and bool not isinstance(resource[k], text_type)): resource[k] = text_type(resource[k])
def _cleanuped_list(self, resource): for k, v in enumerate(resource): if not v: continue if isinstance(v, list): self._cleanuped_list(v) elif isinstance(v, dict): self._cleanuped_dict(v) elif (not isinstance(v, int) and # integer and bool not isinstance(v, text_type)): resource[k] = text_type(v)
def test_create_raises_UnknownServiceError(self): _ctx = self._prepare_context(ALIAS_TH, NODE_PROPERTIES) with self.assertRaises(UnknownServiceError) as error: alias.create(ctx=_ctx, resource_config=None, iface=None) self.assertEqual( text_type(error.exception), "Unknown service: 'kms'. Valid service names are: ['rds']" ) self.fake_boto.assert_called_with('kms', **CLIENT_CONFIG)
def test(_value): if isinstance(_value, datetime): return text_type(_value) elif isinstance(_value, list): for _value_item in _value: i = _value.index(_value_item) _value[i] = test(_value_item) return _value elif isinstance(_value, dict): for _value_key, _value_item in _value.items(): _value[_value_key] = test(_value_item) return _value else: return _value
def create(ctx, iface, resource_config, **_): '''Creates an AWS Route53 Hosted Zone''' # Build API params params = \ dict() if not resource_config else resource_config.copy() if iface.resource_id: params.update({'Name': iface.resource_id}) if not params.get('CallerReference'): params.update(dict(CallerReference=text_type(ctx.instance.id))) # Actually create the resource create_response = iface.create(params)['HostedZone']['Id'] iface.update_resource_id(create_response) utils.update_resource_id(ctx.instance, create_response) utils.update_resource_arn(ctx.instance, create_response)
def start(ctx, iface, resource_config, **_): '''Updates an AWS RDS Instance Runtime Properties''' db_instance = iface.properties for key, value in db_instance.items(): if key == 'DBInstanceIdentifier': iface.update_resource_id(value) utils.update_resource_id(ctx.instance, value) continue elif key == 'DBInstanceArn': utils.update_resource_arn(ctx.instance, value) continue elif isinstance(value, datetime): value = text_type(value) ctx.instance.runtime_properties[key] = value
def delete(ctx, iface, resource_config, **_): """Deletes an AWS EFS File System""" # Create a copy of the resource config for clean manipulation. params = \ dict() if not resource_config else resource_config.copy() file_system_id = params.get(FILESYSTEM_ID) if not file_system_id: params[FILESYSTEM_ID] = iface.resource_id # Actually delete the resource try: iface.delete(params) except ClientError as e: return ctx.operation.retry(text_type(e))
def test_prepare_assoc_Role_NonRecoverableError(self): _source_ctx, _target_ctx, _ctx = self._create_common_relationships( 'test_prepare_assoc', source_type_hierarchy=INSTANCE_READ_REPLICA_TH, target_type_hierarchy=[ 'cloudify.nodes.Root', 'cloudify.nodes.aws.iam.Role' ]) current_ctx.set(_ctx) with self.assertRaises(NonRecoverableError) as error: instance_read_replica.prepare_assoc(ctx=_ctx, resource_config=None, iface=None) self.assertEqual(text_type(error.exception), ('Missing required relationship inputs ' + '"iam_role_type_key" and/or "iam_role_id_key".'))
def test_delete_client_error(self): _test_name = 'test_delete' _ctx = self.get_mock_ctx( _test_name, test_properties=NODE_PROPERTIES, test_runtime_properties=RUNTIME_PROPERTIES_AFTER_CREATE, type_hierarchy=OPTION_GROUP_TH) current_ctx.set(_ctx) self.fake_client.delete_option_group = self._gen_client_error( 'test_delete', message='SomeMessage') with self.assertRaises(OperationRetry) as error: option_group.delete(ctx=_ctx, resource_config=None, iface=None) self.assertEqual(text_type(error.exception), 'SomeMessage')
def create(ctx, iface, resource_config, params, **_): '''Creates an AWS ELB load balancer''' # LB attributes are only applied in modify operation. params.pop(LB_ATTR, {}) # Add Subnets subnets_from_params = params.get(SUBNETS, []) subnets = \ utils.find_rels_by_node_type( ctx.instance, SUBNET_TYPE) or utils.find_rels_by_node_name( ctx.instance, SUBNET_TYPE_DEPRECATED) for subnet in subnets: subnet_id = \ subnet.target.instance.runtime_properties[EXTERNAL_RESOURCE_ID] subnets_from_params.append(subnet_id) params[SUBNETS] = subnets_from_params # Add Security Groups secgroups_from_params = params.get(SECGROUPS, []) secgroups = \ utils.find_rels_by_node_type( ctx.instance, SECGROUP_TYPE) or \ utils.find_rels_by_node_type( ctx.instance, SECGROUP_TYPE_DEPRECATED) for secgroup in secgroups: secgroup_id = \ secgroup.target.instance.runtime_properties[EXTERNAL_RESOURCE_ID] secgroups_from_params.append(secgroup_id) params[SECGROUPS] = secgroups_from_params # Actually create the resource output = iface.create(params) lb_id = output['LoadBalancers'][0][RESOURCE_NAME] iface.resource_id = lb_id try: utils.update_resource_id( ctx.instance, lb_id) utils.update_resource_arn( ctx.instance, output['LoadBalancers'][0][LB_ARN]) except (IndexError, KeyError) as e: raise NonRecoverableError( '{0}: {1} or {2} not located in response: {3}'.format( text_type(e), RESOURCE_NAME, LB_ARN, output))
def delete(ctx, iface, resource_config, **_): """Deletes an AWS EC2 ElasticIP""" # Create a copy of the resource config for clean manipulation. params = \ dict() if not resource_config else resource_config.copy() allocation_id = params.get(ALLOCATION_ID) if not allocation_id: allocation_id = \ ctx.instance.runtime_properties.get( 'allocation_id') elasticip_id = params.get(ELASTICIP_ID) if not elasticip_id: elasticip_id = iface.resource_id if allocation_id: params[ALLOCATION_ID] = allocation_id try: del params[ELASTICIP_ID] except KeyError: pass elif elasticip_id: params[ELASTICIP_ID] = elasticip_id try: del params[ALLOCATION_ID] except KeyError: pass if ctx.node.properties.get('use_unassociated_addresses', False): address = ctx.instance.runtime_properties.pop('unassociated_address', None) if address: ctx.logger.info( 'Not deleting address {address}'.format(address=address)) return try: iface.delete(params) except ClientError as e: if 'AuthFailure' is text_type(e): raise OperationRetry('Address has not released yet.') else: pass
def make_client_call(self, client_method_name, client_method_args=None, log_response=True, fatal_handled_exceptions=FATAL_EXCEPTIONS): """ :param client_method_name: A method on self.client. :param client_method_args: Optional Args. :param log_response: Whether to log API response. :param fatal_handled_exceptions: exceptions to fail on. :return: Either Exception class or successful response content. """ type_name = getattr(self, 'type_name') self.logger.debug('Calling {0} method {1} with parameters: {2}'.format( type_name, client_method_name, client_method_args)) client_method = getattr(self.client, client_method_name) if not client_method: return try: if isinstance(client_method_args, dict): res = client_method(**client_method_args) elif isinstance(client_method_args, list): res = client_method(*client_method_args) else: res = client_method_args() except fatal_handled_exceptions as error: _, _, tb = sys.exc_info() if isinstance(error, ClientError) and hasattr(error, 'message'): message = error.message + NTP_NOTE else: message = 'API error encountered: {}'.format(error) raise NonRecoverableError( text_type(message), causes=[exception_to_error_cause(error, tb)]) else: if log_response: self.logger.debug('Response: {0}'.format(res)) return res
def get_resource_string(node=None, instance=None, property_key=None, attribute_key=None): ''' Gets a string of a Cloudify node and/or instance, searching both properties and runtime properties (attributes). :param `cloudify.context.NodeContext` node: Cloudify node. :param `cloudify.context.NodeInstanceContext` instance: Cloudify node instance. ''' node = node if node else ctx.node instance = instance if instance else ctx.instance props = node.properties if node else {} runtime_props = instance.runtime_properties if instance else {} # Search instance runtime properties first, then the node properties value = runtime_props.get(attribute_key, props.get(property_key)) return text_type(value) if value else None
def test_delete_unexpected_client_error(self): _test_name = 'test_delete' _ctx = self.get_mock_ctx( _test_name, test_properties=NODE_PROPERTIES, test_runtime_properties=RUNTIME_PROPERTIES_AFTER_CREATE, type_hierarchy=OPTION_GROUP_TH) current_ctx.set(_ctx) self.fake_client.delete_option_group = self._gen_client_error( 'test_delete', message='SomeMessage', code='InvalidFault') with self.assertRaises(ClientError) as error: option_group.delete(ctx=_ctx, resource_config=None, iface=None) self.assertEqual( text_type(error.exception), ('An error occurred (InvalidFault) when calling the ' + 'client_error_test_delete operation: SomeMessage'))
def test_create_raises_UnknownServiceError(self): _ctx = self.get_mock_ctx( 'test_create', test_properties=NODE_PROPERTIES, test_runtime_properties=RUNTIME_PROPERTIES, type_hierarchy=QUEUE_TH ) current_ctx.set(_ctx) with self.assertRaises(UnknownServiceError) as error: queue.create(ctx=_ctx, resource_config=None, iface=None) self.assertEqual( text_type(error.exception), "Unknown service: 'sqs'. Valid service names are: ['rds']" ) self.fake_boto.assert_called_with('sqs', **CLIENT_CONFIG)
def test_class_properties(self): effect = self.get_client_error_exception(name='EC2 Image') self.image.client = self.make_client_function('describe_images', side_effect=effect) res = self.image.properties self.assertIsNone(res) value = {} self.image.client = self.make_client_function('describe_images', return_value=value) with self.assertRaises(NonRecoverableError) as e: self.image.properties self.assertEqual(text_type(e.exception), u"Found no AMIs matching provided filters.") value = {IMAGES: [{IMAGE_ID: 'test_name'}]} self.image.client = self.make_client_function('describe_images', return_value=value) res = self.image.properties self.assertEqual(res[IMAGE_ID], 'test_name')
def test_create_raises_UnknownServiceError(self): _test_name = 'test_create_UnknownServiceError' _test_runtime_properties = {'resource_config': {}} _ctx = self.get_mock_ctx( _test_name, test_properties=NODE_PROPERTIES, test_runtime_properties=_test_runtime_properties, type_hierarchy=INSTANCE_READ_REPLICA_TH) current_ctx.set(_ctx) with self.assertRaises(UnknownServiceError) as error: instance_read_replica.create(ctx=_ctx, resource_config=None, iface=None) self.assertEqual( text_type(error.exception), "Unknown service: 'rds'. Valid service names are: ['rds']") self.fake_boto.assert_called_with('rds', **CLIENT_CONFIG)
def test_class_status(self): value = {} self.image.client = self.make_client_function('describe_images', return_value=value) with self.assertRaises(NonRecoverableError) as e: self.image.status self.assertEqual(text_type(e.exception), u"Found no AMIs matching provided filters.") value = {IMAGES: [None]} self.image.client = self.make_client_function('describe_images', return_value=value) res = self.image.status self.assertIsNone(res) value = {IMAGES: [{IMAGE_ID: 'test_name', 'State': 'available'}]} self.image.client = self.make_client_function('describe_images', return_value=value) res = self.image.status self.assertEqual(res, 'available')
def create(ctx, iface, resource_config, **_): '''Creates an AWS RDS Instance''' # Build API params params = \ dict() if not resource_config else resource_config.copy() params.update(dict(DBInstanceIdentifier=iface.resource_id)) # Actually create the resource res = iface.create(params) db_instance = res['DBInstance'] for key, value in db_instance.items(): if key == 'DBInstanceIdentifier': iface.update_resource_id(value) utils.update_resource_id(ctx.instance, value) continue elif key == 'DBInstanceArn': utils.update_resource_arn(ctx.instance, value) continue elif isinstance(value, datetime): value = text_type(value) ctx.instance.runtime_properties[key] = value
def delete(ctx, iface, resource_config, **_): """Deletes an AWS ELB classic policy""" # Create a copy of the resource config for clean manipulation. params = \ dict() if not resource_config else resource_config.copy() lb = params.get(LB_NAME) or ctx.instance.runtime_properties.get(LB_NAME) policy = \ params.get(RESOURCE_NAME) or \ ctx.instance.runtime_properties.get(RESOURCE_NAME) lb_delete_params = {LB_NAME: lb, RESOURCE_NAME: policy} try: iface.delete(lb_delete_params) except ClientError as e: if _.get('force'): raise OperationRetry('Retrying: {0}'.format(text_type(e))) pass