def test_member_putpost(self): API = BASE_URL + '/v1/service/member' service = config.server.service member_id = uuid4() # HACK: MemberSecret takes an Account instance as third parameter but # we use a Service instance instead service.paths.account = 'pod' secret = MemberSecret(member_id, SERVICE_ID, service) csr = secret.create_csr() csr = csr.public_bytes(serialization.Encoding.PEM) response = requests.post(API, json={'csr': str(csr, 'utf-8')}, headers=None) self.assertEqual(response.status_code, 201) data = response.json() self.assertTrue('signed_cert' in data) self.assertTrue('cert_chain' in data) self.assertTrue('service_data_cert_chain' in data) signed_secret = MemberSecret(member_id, SERVICE_ID, service) signed_secret.from_string(data['signed_cert'], certchain=data['cert_chain']) service_data_cert_chain = secret.from_string( # noqa: F841 data['service_data_cert_chain']) membersecret_commonname = Secret.extract_commonname(signed_secret.cert) memberscasecret_commonname = Secret.extract_commonname( signed_secret.cert_chain[0]) # PUT, with auth # In the PUT body we put the member data secret as a service may # have use for it in the future. member_data_secret = MemberDataSecret(member_id, SERVICE_ID, service) csr = member_data_secret.create_csr() cert_chain = service.members_ca.sign_csr(csr) member_data_secret.from_signed_cert(cert_chain) member_data_certchain = member_data_secret.certchain_as_pem() headers = { 'X-Client-SSL-Verify': 'SUCCESS', 'X-Client-SSL-Subject': f'CN={membersecret_commonname}', 'X-Client-SSL-Issuing-CA': f'CN={memberscasecret_commonname}' } response = requests.put(f'{API}/version/1', headers=headers, json={'certchain': member_data_certchain}) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(data['ipv4_address'], '127.0.0.1') self.assertEqual(data['ipv6_address'], None)
async def _create_secret(self, secret_cls: Callable, issuing_ca: Secret ) -> Secret: ''' Abstraction for creating secrets for the Member class to avoid repetition of code for creating the various member secrets of the Service class :param secret_cls: callable for one of the classes derived from byoda.util.secrets.Secret :param issuing_ca: ca to sign the cert locally, instead of requiring the service to sign the cert request :raises: ValueError, NotImplementedError ''' if not self.member_id: raise ValueError( 'Member_id for the account has not been defined' ) secret = secret_cls( self.member_id, self.service_id, account=self.account ) if await secret.cert_file_exists(): raise ValueError( f'Cert for {type(secret)} for service {self.service_id} and ' f'member {self.member_id} already exists' ) if await secret.private_key_file_exists(): raise ValueError( f'Private key for {type(secret)} for service {self.service_id}' f' and member {self.member_id} already exists' ) if not issuing_ca: if secret_cls != MemberSecret and secret_cls != MemberDataSecret: raise ValueError( f'No issuing_ca was provided for creating a ' f'{type(secret_cls)}' ) else: # Get the CSR signed, the resulting cert saved to disk # and used to register with both the network and the service await self.register(secret) else: csr = secret.create_csr() issuing_ca.review_csr(csr, source=CsrSource.LOCAL) certchain = issuing_ca.sign_csr(csr) secret.from_signed_cert(certchain) await secret.save(password=self.private_key_password) return secret
async def put_member( request: Request, schema_version: int, certchain: CertChainRequestModel, auth: MemberRequestAuthFast = Depends(MemberRequestAuthFast)): ''' Registers a known pod with its IP address and its data cert ''' _LOGGER.debug(f'PUT Member API called from {request.client.host}') await auth.authenticate() network = config.server.network service = config.server.service if service.service_id != auth.service_id: _LOGGER.debug( f'Service ID {service.service_id} of PUT call does not match ' f'service ID {auth.service_id} in client cert') raise HTTPException(404) paths = copy(network.paths) paths.account = 'pod' # We use a trick here to make sure we get unique filenames for the # member data secret by replacing the service_id in the template # with the member_id member_data_secret = Secret(cert_file=paths.get( Paths.MEMBER_DATA_CERT_FILE, service_id=auth.member_id), key_file=paths.get(Paths.MEMBER_DATA_KEY_FILE, service_id=auth.member_id), storage_driver=network.paths.storage_driver) # from_string() concats the cert and the certchain together # so we can use it here with just providing the certchain parameter member_data_secret.from_string(certchain.certchain) await member_data_secret.save(overwrite=True) config.server.member_db.add_meta(auth.member_id, auth.remote_addr, schema_version, certchain.certchain, MemberStatus.REGISTERED) _LOGGER.debug(f'Updating registration for member_id {auth.member_id} with ' f'schema version {schema_version} and ' f'remote address {auth.remote_addr}') return {'ipv4_address': auth.remote_addr}
def __init__(self, api: str, secret: Secret = None, service_id: int = None): ''' Maintains a pool of connections for different destinations :raises: ValueError is service_id is specified for a secret that is not a MemberSecret ''' server = config.server # We maintain a cache of sessions based on the authentication # requirements of the remote host and whether to use for verifying # the TLS server cert the root CA of the network or the regular CAs. self.session = None if not secret: pool = 'noauth' elif isinstance(secret, ServiceSecret): pool = f'service-{service_id}' elif isinstance(secret, MemberSecret): pool = 'member' elif isinstance(secret, AccountSecret): pool = 'account' else: raise ValueError('Secret must be either an account-, member- or ' f'service-secret, not {type(secret)}') if pool not in config.client_pools: if api.startswith(f'https://dir'): # For calls by Accounts and Services to the directory server, # we do not have to set the root CA as the directory server # uses a Let's Encrypt cert _LOGGER.debug('No using byoda certchain for server cert ' f'verification of {api}') self.ssl_context = ssl.create_default_context() else: filepath = (server.network.paths._root_directory + '/' + server.network.root_ca.cert_file) self.ssl_context = ssl.create_default_context(cafile=filepath) _LOGGER.debug(f'Set server cert validation to {filepath}') if secret: key_path = secret.save_tmp_private_key() cert_filepath = (server.network.paths.root_directory + '/' + secret.cert_file) _LOGGER.debug( f'Setting client cert/key to {cert_filepath}, {key_path}') self.ssl_context.load_cert_chain(cert_filepath, key_path) timeout = aiohttp.ClientTimeout(total=10) self.session = aiohttp.ClientSession(timeout=timeout) config.client_pools[type(secret)] = self.session else: self.session = config.client_pools[type(secret)]
async def _create_secret(network: str, secret_cls: Callable, issuing_ca: Secret, paths: Paths, password: str): ''' Abstraction helper for creating secrets for a Network to avoid repetition of code for creating the various member secrets of the Network class :param secret_cls: callable for one of the classes derived from byoda.util.secrets.Secret :raises: ValueError ''' if not network: raise ValueError( 'Name and service_id of the service have not been defined' ) if not issuing_ca: raise ValueError( f'No issuing_ca was provided for creating a ' f'{type(secret_cls)}' ) secret = secret_cls(paths=paths) if await secret.cert_file_exists(): await secret.load(password=password) return secret # TODO: SECURITY: add constraints csr = secret.create_csr() issuing_ca.review_csr(csr, source=CsrSource.LOCAL) certchain = issuing_ca.sign_csr(csr) secret.from_signed_cert(certchain) await secret.save(password=password) return secret
def sign(self, csr: str, id_type: IdType, remote_addr: IpAddress) -> CertChain: ''' Evaluate a CSR and sign it :param csr: the Certificate Signing Request :param id_type: what entity is the CSR for, client, service or member :param remote_addr: the originating IP address for the CSR :returns: the signed certificate and its certchain :raises: KeyError if the Certificate Name is not acceptable, ValueError if there is something else unacceptable in the CSR ''' if type(csr) not in (str, bytes): raise ValueError('CSR must be a string or a byte array') cert_auth = self.ca_secret csr = Secret.csr_from_string(csr) extension = csr.extensions.get_extension_for_class( x509.BasicConstraints) if not cert_auth.signs_ca_certs and extension.value.ca: raise ValueError('Certificates with CA bits set are not permitted') entity_id = cert_auth.review_csr(csr) if entity_id.id_type == IdType.SERVICE: raise NotImplementedError( 'Service certs are not supported for this API, ' 'only ServiceCA certs') # TODO: add check on whether the UUID is already in use certchain = cert_auth.sign_csr(csr, 365 * 3) id_type = entity_id.id_type.value.strip('-') _LOGGER.info(f'Signed the CSR for {entity_id.id} for IdType {id_type} ' f'received from IP {str(remote_addr)}') return certchain
async def test_network_service_creation(self): API = BASE_URL + '/v1/network/service' # We can not use deepcopy here so do two copies network = copy(config.server.network) network.paths = copy(config.server.network.paths) network.paths._root_directory = SERVICE_DIR if not await network.paths.secrets_directory_exists(): await network.paths.create_secrets_directory() service_id = SERVICE_ID serviceca_secret = ServiceCaSecret(service='dir_api_test', service_id=service_id, network=network) csr = serviceca_secret.create_csr() csr = csr.public_bytes(serialization.Encoding.PEM) response = requests.post(API, json={'csr': str(csr, 'utf-8')}) self.assertEqual(response.status_code, 201) data = response.json() issuing_ca_cert = x509.load_pem_x509_certificate( # noqa:F841 data['cert_chain'].encode()) serviceca_cert = x509.load_pem_x509_certificate( # noqa:F841 data['signed_cert'].encode()) # TODO: populate a secret from a CertChain serviceca_secret.cert = serviceca_cert serviceca_secret.cert_chain = [issuing_ca_cert] network_data_cert = x509.load_pem_x509_certificate( # noqa:F841 data['network_data_cert_chain'].encode()) # Check that the service CA public cert was written to the network # directory of the dirserver testsecret = ServiceCaSecret(service='dir_api_test', service_id=service_id, network=config.server.network) await testsecret.load(with_private_key=False) service_secret = ServiceSecret('dir_api_test', service_id, network) service_csr = service_secret.create_csr() certchain = serviceca_secret.sign_csr(service_csr) service_secret.from_signed_cert(certchain) await service_secret.save() service_cn = Secret.extract_commonname(certchain.signed_cert) serviceca_cn = Secret.extract_commonname(serviceca_cert) # Create and register the the public cert of the data secret, # which the directory server needs to validate the service signature # of the schema for the service service_data_secret = ServiceDataSecret('dir_api_test', service_id, network) service_data_csr = service_data_secret.create_csr() data_certchain = serviceca_secret.sign_csr(service_data_csr) service_data_secret.from_signed_cert(data_certchain) await service_data_secret.save() headers = { 'X-Client-SSL-Verify': 'SUCCESS', 'X-Client-SSL-Subject': f'CN={service_cn}', 'X-Client-SSL-Issuing-CA': f'CN={serviceca_cn}' } data_certchain = service_data_secret.certchain_as_pem() response = requests.put(API + '/service_id/' + str(service_id), headers=headers, json={'certchain': data_certchain}) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(data['ipv4_address'], '127.0.0.1') # Send the service schema with open(DEFAULT_SCHEMA) as file_desc: data = file_desc.read() schema_data = orjson.loads(data) schema_data['service_id'] = service_id schema_data['version'] = 1 schema = Schema(schema_data) schema.create_signature(service_data_secret, SignatureType.SERVICE) headers = { 'X-Client-SSL-Verify': 'SUCCESS', 'X-Client-SSL-Subject': f'CN={service_cn}', 'X-Client-SSL-Issuing-CA': f'CN={serviceca_cn}' } response = requests.patch(API + f'/service_id/{service_id}', headers=headers, json=schema.json_schema) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(data['status'], 'ACCEPTED') self.assertEqual(len(data['errors']), 0) # Get the fully-signed data contract for the service API = BASE_URL + '/v1/network/service' response = requests.get(API + f'/service_id/{service_id}') self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(len(data), 10) self.assertEqual(data['service_id'], SERVICE_ID) self.assertEqual(data['version'], 1) self.assertEqual(data['name'], 'dummyservice') self.assertEqual(len(data['signatures']), 2) schema = Schema(data) # Get the list of service summaries API = BASE_URL + '/v1/network/services' response = requests.get(API) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(len(data), 1) service_summary = data['service_summaries'][0] self.assertEqual(service_summary['service_id'], SERVICE_ID) self.assertEqual(service_summary['version'], 1) self.assertEqual(service_summary['name'], 'dummyservice') # Now test membership registration against the directory server API = BASE_URL + '/v1/network/member' headers = { 'X-Client-SSL-Verify': 'SUCCESS', 'X-Client-SSL-Subject': f'CN={uuid4()}.members-{service_id}.{network.name}', 'X-Client-SSL-Issuing-CA': f'CN=members-ca.members-ca-{service_id}.{network.name}' } response = requests.put(API, headers=headers) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(data['ipv4_address'], '127.0.0.1')
async def get_csr_signature(self, secret: Secret, csr: CSR, issuing_ca: CaSecret, private_key_password: str = None) -> None: ''' Gets the signed cert(chain) for the CSR and saves returned cert and the existing private key. If the issuing_ca parameter is specified then the CSR will be signed directly by the private key of the issuing_ca, otherwise the CSR will be send in a POST /api/v1/network/service API call to the directory server of the network ''' if (isinstance(secret, ServiceCaSecret) and ( self.registration_status != RegistrationStatus.Unknown or await secret.cert_file_exists())): # TODO: support renewal of ServiceCA cert raise ValueError('ServiceCA cert has already been signed') if issuing_ca: # We have the private key of the issuing CA so can # sign ourselves and be done with it issuing_ca.review_csr(csr, source=CsrSource.LOCAL) certchain = issuing_ca.sign_csr(csr) secret.from_signed_cert(certchain) await secret.save(password=private_key_password, overwrite=False) # We do not set self.registration_status as locally signing # does not provide information about the status of service in # the network return if not isinstance(secret, ServiceCaSecret): raise ValueError( f'No issuing_ca was provided for creating a ' f'{type(secret)}' ) # We have to get our signature from the directory server data = { 'csr': str( csr.public_bytes(serialization.Encoding.PEM), 'utf-8' ) } url = self.paths.get(Paths.NETWORKSERVICE_POST_API) response = await RestApiClient.call( url, HttpMethod.POST, data=data ) if response.status != 201: raise ValueError( f'Failed to POST to API {Paths.NETWORKSERVICE_API}: ' f'{response.status}' ) data = await response.json() secret.from_string(data['signed_cert'] + data['cert_chain']) self.registration_status = RegistrationStatus.CsrSigned await secret.save(password=private_key_password, overwrite=False) # Every time we receive the network data cert, we # save it as it could have changed since the last time we # got it network = config.server.network if not network.data_secret: network.data_secret = NetworkDataSecret(network.paths) network.data_secret.from_string(data['network_data_cert_chain']) await network.data_secret.save(overwrite=True)
async def _create_secret(self, secret_cls: Callable, issuing_ca: Secret ) -> Secret: ''' Abstraction for creating secrets for the Service class to avoid repetition of code for creating the various member secrets of the Service class :param secret_cls: callable for one of the classes derived from byoda.util.secrets.Secret :raises: ValueError, NotImplementedError ''' if not self.account_id: raise ValueError( 'Account_id for the account has not been defined' ) secret = secret_cls( self.account, self.account_id, network=self.network ) if await secret.cert_file_exists(): raise ValueError( f'Cert for {type(secret)} for account_id {self.account_id} ' 'already exists' ) if await secret.private_key_file_exists(): raise ValueError( f'Private key for {type(secret)} for account_id ' f'{self.account_id} already exists' ) if not issuing_ca: if secret_cls != AccountSecret and secret_cls != AccountDataSecret: raise ValueError( f'No issuing_ca was provided for creating a ' f'{type(secret_cls)}' ) else: csr = secret.create_csr(self.account_id) payload = {'csr': secret.csr_as_pem(csr).decode('utf-8')} url = self.paths.get(Paths.NETWORKACCOUNT_API) # TODO: Refactor to use RestClientApi _LOGGER.debug(f'Getting CSR signed from {url}') resp = await RestApiClient.call( url, method=HttpMethod.POST, data=payload ) if resp.status != 201: raise RuntimeError('Certificate signing request failed') cert_data = await resp.json() secret.from_string( cert_data['signed_cert'], certchain=cert_data['cert_chain'] ) else: csr = secret.create_csr() issuing_ca.review_csr(csr, source=CsrSource.LOCAL) certchain = issuing_ca.sign_csr(csr) secret.from_signed_cert(certchain) await secret.save(password=self.private_key_password) return secret
async def post_service(request: Request, csr: CertSigningRequestModel, auth: ServiceRequestOptionalAuthFast = Depends( ServiceRequestOptionalAuthFast), db_session=Depends(asyncdb_session)): ''' Submit a Certificate Signing Request for the ServiceCA certificate and get the cert signed by the network services CA This API is called by services This API does not require authentication, it needs to be rate limited by the reverse proxy ''' _LOGGER.debug(f'POST Service API called from {request.client.host}') network = config.server.network dnsdb: DnsDb = config.server.network.dnsdb # Authorization csr_x509: x509 = Secret.csr_from_string(csr.csr) common_name = Secret.extract_commonname(csr_x509) try: entity_id = NetworkServicesCaSecret.review_commonname_by_parameters( common_name, network.name) except PermissionError: raise HTTPException(status_code=401, detail=f'Invalid common name {common_name} in CSR') except (ValueError, KeyError): raise HTTPException( status_code=400, detail=( f'error when reviewing the common name {common_name} in your ' 'CSR')) try: await dnsdb.lookup_fqdn(common_name, DnsRecordType.A, db_session) dns_exists = True except KeyError: dns_exists = False if auth.is_authenticated: if auth.auth_source != AuthSource.CERT: raise HTTPException( status_code=401, detail=('When used with credentials, this API requires ' 'authentication with TLS client cert')) if auth.id_type != IdType.SERVICE: raise HTTPException( status_code=401, detail='A TLS cert of a service is required for this API') if auth.service_id != entity_id.service_id: raise HTTPException( status_code=403, detail=( f'Client auth for service id {auth.service_id} does not ' f'match CSR for service_id {entity_id.service_id}')) else: if dns_exists: raise HTTPException( status_code=403, detail=( 'CSR is for existing service, must use TLS Client cert ' 'for authentication')) # End of Authorization # The Network Services CA signs the CSRs for Service CAs certstore = CertStore(network.services_ca) certchain = certstore.sign(csr.csr, IdType.SERVICE_CA, request.client.host) # Create the service and add it to the network service = Service(network=network, service_id=entity_id.service_id) network.services[entity_id.service_id] = service # Get the certs as strings so we can return them signed_cert = certchain.cert_as_string() cert_chain = certchain.cert_chain_as_string() # We save the public key in the network directory tree. Not sure # if we actually need to do this as we can check any cert of the service # and its members through the cert chain that is chained to the network # root CA service.service_ca = ServiceCaSecret(None, service.service_id, network) service.service_ca.cert = certchain.signed_cert service.service_ca.cert_chain = certchain.cert_chain # If someone else already registered a Service then saving the cert will # raise an exception try: await service.service_ca.save(overwrite=False) except PermissionError: raise HTTPException(409, 'Service CA certificate already exists') # We create the DNS entry if it not exists yet to make sure there is no # race condition between submitting the CSR through the POST API and # registering the service server through the PUT API if not dns_exists: await dnsdb.create_update(None, IdType.SERVICE, auth.remote_addr, db_session, service_id=entity_id.service_id) data_cert = network.data_secret.cert_as_pem() return { 'signed_cert': signed_cert, 'cert_chain': cert_chain, 'network_data_cert_chain': data_cert, }
async def post_member(request: Request, csr: CertSigningRequestModel, auth: MemberRequestAuthOptionalFast = Depends( MemberRequestAuthOptionalFast)): ''' Submit a Certificate Signing Request for the Member certificate and get the cert signed by the Service Members CA This API is called by pods This API does not require authentication, it needs to be rate limited by the reverse proxy (TODO: security) ''' _LOGGER.debug(f'POST Member API called from {request.client.host}') await auth.authenticate() server: ServiceServer = config.server service: Service = server.service network: Network = server.network # Authorization csr_x509: x509 = Secret.csr_from_string(csr.csr) common_name = Secret.extract_commonname(csr_x509) try: entity_id = MembersCaSecret.review_commonname_by_parameters( common_name, network.name, service.service_id) except PermissionError: raise HTTPException(status_code=401, detail=f'Invalid common name {common_name} in CSR') except (ValueError, KeyError): raise HTTPException( status_code=400, detail=( f'error when reviewing the common name {common_name} in your ' 'CSR')) if auth.is_authenticated: if auth.auth_source != AuthSource.CERT: raise HTTPException( status_code=401, detail=('When used with credentials, this API requires ' 'authentication with a TLS client cert')) if entity_id.id_type != IdType.MEMBER: raise HTTPException( status_code=403, detail='A TLS cert of a member must be used with this API') _LOGGER.debug(f'Signing csr for existing member {entity_id.id}') else: # TODO: security: consider tracking member UUIDs to avoid # race condition between CSR signature and member registration # with the Directory server ips = server.dns_resolver.resolve(common_name) if ips: _LOGGER.debug('Attempt to submit CSR for existing member without ' 'authentication') raise HTTPException( status_code=401, detail=( 'Must use TLS client cert when renewing a member cert')) _LOGGER.debug(f'Signing csr for new member {entity_id.id}') # End of Authorization if entity_id.service_id is None: raise ValueError(f'No service id found in common name {common_name}') if entity_id.service_id != service.service_id: raise HTTPException( 404, f'Incorrect service_id in common name {common_name}') # The Network Services CA signs the CSRs for Service CAs certstore = CertStore(service.members_ca) certchain = certstore.sign(csr.csr, IdType.MEMBER, request.client.host) # Get the certs as strings so we can return them signed_cert = certchain.cert_as_string() cert_chain = certchain.cert_chain_as_string() service_data_cert_chain = service.data_secret.cert_as_pem() _LOGGER.info(f'Signed certificate with commonname {common_name}') config.server.member_db.add_meta(entity_id.id, request.client.host, None, cert_chain, MemberStatus.SIGNED) return { 'signed_cert': signed_cert, 'cert_chain': cert_chain, 'service_data_cert_chain': service_data_cert_chain, }