def test_member_putpost(self):
API = BASE_URL + '/v1/service/member'
service = config.server.service
member_id = uuid4()
# HACK: MemberSecret takes an Account instance as third parameter but
# we use a Service instance instead
service.paths.account = 'pod'
secret = MemberSecret(member_id, SERVICE_ID, service)
csr = secret.create_csr()
csr = csr.public_bytes(serialization.Encoding.PEM)
response = requests.post(API,
json={'csr': str(csr, 'utf-8')},
headers=None)
self.assertEqual(response.status_code, 201)
data = response.json()
self.assertTrue('signed_cert' in data)
self.assertTrue('cert_chain' in data)
self.assertTrue('service_data_cert_chain' in data)
signed_secret = MemberSecret(member_id, SERVICE_ID, service)
signed_secret.from_string(data['signed_cert'],
certchain=data['cert_chain'])
service_data_cert_chain = secret.from_string( # noqa: F841
data['service_data_cert_chain'])
membersecret_commonname = Secret.extract_commonname(signed_secret.cert)
memberscasecret_commonname = Secret.extract_commonname(
signed_secret.cert_chain[0])
# PUT, with auth
# In the PUT body we put the member data secret as a service may
# have use for it in the future.
member_data_secret = MemberDataSecret(member_id, SERVICE_ID, service)
csr = member_data_secret.create_csr()
cert_chain = service.members_ca.sign_csr(csr)
member_data_secret.from_signed_cert(cert_chain)
member_data_certchain = member_data_secret.certchain_as_pem()
headers = {
'X-Client-SSL-Verify': 'SUCCESS',
'X-Client-SSL-Subject': f'CN={membersecret_commonname}',
'X-Client-SSL-Issuing-CA': f'CN={memberscasecret_commonname}'
}
response = requests.put(f'{API}/version/1',
headers=headers,
json={'certchain': member_data_certchain})
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertEqual(data['ipv4_address'], '127.0.0.1')
self.assertEqual(data['ipv6_address'], None)
async def _create_secret(self, secret_cls: Callable, issuing_ca: Secret
) -> Secret:
'''
Abstraction for creating secrets for the Member class to avoid
repetition of code for creating the various member secrets of the
Service class
:param secret_cls: callable for one of the classes derived from
byoda.util.secrets.Secret
:param issuing_ca: ca to sign the cert locally, instead of requiring
the service to sign the cert request
:raises: ValueError, NotImplementedError
'''
if not self.member_id:
raise ValueError(
'Member_id for the account has not been defined'
)
secret = secret_cls(
self.member_id, self.service_id, account=self.account
)
if await secret.cert_file_exists():
raise ValueError(
f'Cert for {type(secret)} for service {self.service_id} and '
f'member {self.member_id} already exists'
)
if await secret.private_key_file_exists():
raise ValueError(
f'Private key for {type(secret)} for service {self.service_id}'
f' and member {self.member_id} already exists'
)
if not issuing_ca:
if secret_cls != MemberSecret and secret_cls != MemberDataSecret:
raise ValueError(
f'No issuing_ca was provided for creating a '
f'{type(secret_cls)}'
)
else:
# Get the CSR signed, the resulting cert saved to disk
# and used to register with both the network and the service
await self.register(secret)
else:
csr = secret.create_csr()
issuing_ca.review_csr(csr, source=CsrSource.LOCAL)
certchain = issuing_ca.sign_csr(csr)
secret.from_signed_cert(certchain)
await secret.save(password=self.private_key_password)
return secret
async def put_member(
request: Request,
schema_version: int,
certchain: CertChainRequestModel,
auth: MemberRequestAuthFast = Depends(MemberRequestAuthFast)):
'''
Registers a known pod with its IP address and its data cert
'''
_LOGGER.debug(f'PUT Member API called from {request.client.host}')
await auth.authenticate()
network = config.server.network
service = config.server.service
if service.service_id != auth.service_id:
_LOGGER.debug(
f'Service ID {service.service_id} of PUT call does not match '
f'service ID {auth.service_id} in client cert')
raise HTTPException(404)
paths = copy(network.paths)
paths.account = 'pod'
# We use a trick here to make sure we get unique filenames for the
# member data secret by replacing the service_id in the template
# with the member_id
member_data_secret = Secret(cert_file=paths.get(
Paths.MEMBER_DATA_CERT_FILE, service_id=auth.member_id),
key_file=paths.get(Paths.MEMBER_DATA_KEY_FILE,
service_id=auth.member_id),
storage_driver=network.paths.storage_driver)
# from_string() concats the cert and the certchain together
# so we can use it here with just providing the certchain parameter
member_data_secret.from_string(certchain.certchain)
await member_data_secret.save(overwrite=True)
config.server.member_db.add_meta(auth.member_id, auth.remote_addr,
schema_version, certchain.certchain,
MemberStatus.REGISTERED)
_LOGGER.debug(f'Updating registration for member_id {auth.member_id} with '
f'schema version {schema_version} and '
f'remote address {auth.remote_addr}')
return {'ipv4_address': auth.remote_addr}
def __init__(self,
api: str,
secret: Secret = None,
service_id: int = None):
'''
Maintains a pool of connections for different destinations
:raises: ValueError is service_id is specified for a secret that
is not a MemberSecret
'''
server = config.server
# We maintain a cache of sessions based on the authentication
# requirements of the remote host and whether to use for verifying
# the TLS server cert the root CA of the network or the regular CAs.
self.session = None
if not secret:
pool = 'noauth'
elif isinstance(secret, ServiceSecret):
pool = f'service-{service_id}'
elif isinstance(secret, MemberSecret):
pool = 'member'
elif isinstance(secret, AccountSecret):
pool = 'account'
else:
raise ValueError('Secret must be either an account-, member- or '
f'service-secret, not {type(secret)}')
if pool not in config.client_pools:
if api.startswith(f'https://dir'):
# For calls by Accounts and Services to the directory server,
# we do not have to set the root CA as the directory server
# uses a Let's Encrypt cert
_LOGGER.debug('No using byoda certchain for server cert '
f'verification of {api}')
self.ssl_context = ssl.create_default_context()
else:
filepath = (server.network.paths._root_directory + '/' +
server.network.root_ca.cert_file)
self.ssl_context = ssl.create_default_context(cafile=filepath)
_LOGGER.debug(f'Set server cert validation to {filepath}')
if secret:
key_path = secret.save_tmp_private_key()
cert_filepath = (server.network.paths.root_directory + '/' +
secret.cert_file)
_LOGGER.debug(
f'Setting client cert/key to {cert_filepath}, {key_path}')
self.ssl_context.load_cert_chain(cert_filepath, key_path)
timeout = aiohttp.ClientTimeout(total=10)
self.session = aiohttp.ClientSession(timeout=timeout)
config.client_pools[type(secret)] = self.session
else:
self.session = config.client_pools[type(secret)]
async def _create_secret(network: str, secret_cls: Callable,
issuing_ca: Secret, paths: Paths, password: str):
'''
Abstraction helper for creating secrets for a Network to avoid
repetition of code for creating the various member secrets of the
Network class
:param secret_cls: callable for one of the classes derived from
byoda.util.secrets.Secret
:raises: ValueError
'''
if not network:
raise ValueError(
'Name and service_id of the service have not been defined'
)
if not issuing_ca:
raise ValueError(
f'No issuing_ca was provided for creating a '
f'{type(secret_cls)}'
)
secret = secret_cls(paths=paths)
if await secret.cert_file_exists():
await secret.load(password=password)
return secret
# TODO: SECURITY: add constraints
csr = secret.create_csr()
issuing_ca.review_csr(csr, source=CsrSource.LOCAL)
certchain = issuing_ca.sign_csr(csr)
secret.from_signed_cert(certchain)
await secret.save(password=password)
return secret
def sign(self, csr: str, id_type: IdType,
remote_addr: IpAddress) -> CertChain:
'''
Evaluate a CSR and sign it
:param csr: the Certificate Signing Request
:param id_type: what entity is the CSR for, client, service or member
:param remote_addr: the originating IP address for the CSR
:returns: the signed certificate and its certchain
:raises: KeyError if the Certificate Name is not acceptable,
ValueError if there is something else unacceptable in the CSR
'''
if type(csr) not in (str, bytes):
raise ValueError('CSR must be a string or a byte array')
cert_auth = self.ca_secret
csr = Secret.csr_from_string(csr)
extension = csr.extensions.get_extension_for_class(
x509.BasicConstraints)
if not cert_auth.signs_ca_certs and extension.value.ca:
raise ValueError('Certificates with CA bits set are not permitted')
entity_id = cert_auth.review_csr(csr)
if entity_id.id_type == IdType.SERVICE:
raise NotImplementedError(
'Service certs are not supported for this API, '
'only ServiceCA certs')
# TODO: add check on whether the UUID is already in use
certchain = cert_auth.sign_csr(csr, 365 * 3)
id_type = entity_id.id_type.value.strip('-')
_LOGGER.info(f'Signed the CSR for {entity_id.id} for IdType {id_type} '
f'received from IP {str(remote_addr)}')
return certchain
async def test_network_service_creation(self):
API = BASE_URL + '/v1/network/service'
# We can not use deepcopy here so do two copies
network = copy(config.server.network)
network.paths = copy(config.server.network.paths)
network.paths._root_directory = SERVICE_DIR
if not await network.paths.secrets_directory_exists():
await network.paths.create_secrets_directory()
service_id = SERVICE_ID
serviceca_secret = ServiceCaSecret(service='dir_api_test',
service_id=service_id,
network=network)
csr = serviceca_secret.create_csr()
csr = csr.public_bytes(serialization.Encoding.PEM)
response = requests.post(API, json={'csr': str(csr, 'utf-8')})
self.assertEqual(response.status_code, 201)
data = response.json()
issuing_ca_cert = x509.load_pem_x509_certificate( # noqa:F841
data['cert_chain'].encode())
serviceca_cert = x509.load_pem_x509_certificate( # noqa:F841
data['signed_cert'].encode())
# TODO: populate a secret from a CertChain
serviceca_secret.cert = serviceca_cert
serviceca_secret.cert_chain = [issuing_ca_cert]
network_data_cert = x509.load_pem_x509_certificate( # noqa:F841
data['network_data_cert_chain'].encode())
# Check that the service CA public cert was written to the network
# directory of the dirserver
testsecret = ServiceCaSecret(service='dir_api_test',
service_id=service_id,
network=config.server.network)
await testsecret.load(with_private_key=False)
service_secret = ServiceSecret('dir_api_test', service_id, network)
service_csr = service_secret.create_csr()
certchain = serviceca_secret.sign_csr(service_csr)
service_secret.from_signed_cert(certchain)
await service_secret.save()
service_cn = Secret.extract_commonname(certchain.signed_cert)
serviceca_cn = Secret.extract_commonname(serviceca_cert)
# Create and register the the public cert of the data secret,
# which the directory server needs to validate the service signature
# of the schema for the service
service_data_secret = ServiceDataSecret('dir_api_test', service_id,
network)
service_data_csr = service_data_secret.create_csr()
data_certchain = serviceca_secret.sign_csr(service_data_csr)
service_data_secret.from_signed_cert(data_certchain)
await service_data_secret.save()
headers = {
'X-Client-SSL-Verify': 'SUCCESS',
'X-Client-SSL-Subject': f'CN={service_cn}',
'X-Client-SSL-Issuing-CA': f'CN={serviceca_cn}'
}
data_certchain = service_data_secret.certchain_as_pem()
response = requests.put(API + '/service_id/' + str(service_id),
headers=headers,
json={'certchain': data_certchain})
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertEqual(data['ipv4_address'], '127.0.0.1')
# Send the service schema
with open(DEFAULT_SCHEMA) as file_desc:
data = file_desc.read()
schema_data = orjson.loads(data)
schema_data['service_id'] = service_id
schema_data['version'] = 1
schema = Schema(schema_data)
schema.create_signature(service_data_secret, SignatureType.SERVICE)
headers = {
'X-Client-SSL-Verify': 'SUCCESS',
'X-Client-SSL-Subject': f'CN={service_cn}',
'X-Client-SSL-Issuing-CA': f'CN={serviceca_cn}'
}
response = requests.patch(API + f'/service_id/{service_id}',
headers=headers,
json=schema.json_schema)
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertEqual(data['status'], 'ACCEPTED')
self.assertEqual(len(data['errors']), 0)
# Get the fully-signed data contract for the service
API = BASE_URL + '/v1/network/service'
response = requests.get(API + f'/service_id/{service_id}')
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertEqual(len(data), 10)
self.assertEqual(data['service_id'], SERVICE_ID)
self.assertEqual(data['version'], 1)
self.assertEqual(data['name'], 'dummyservice')
self.assertEqual(len(data['signatures']), 2)
schema = Schema(data)
# Get the list of service summaries
API = BASE_URL + '/v1/network/services'
response = requests.get(API)
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertEqual(len(data), 1)
service_summary = data['service_summaries'][0]
self.assertEqual(service_summary['service_id'], SERVICE_ID)
self.assertEqual(service_summary['version'], 1)
self.assertEqual(service_summary['name'], 'dummyservice')
# Now test membership registration against the directory server
API = BASE_URL + '/v1/network/member'
headers = {
'X-Client-SSL-Verify':
'SUCCESS',
'X-Client-SSL-Subject':
f'CN={uuid4()}.members-{service_id}.{network.name}',
'X-Client-SSL-Issuing-CA':
f'CN=members-ca.members-ca-{service_id}.{network.name}'
}
response = requests.put(API, headers=headers)
self.assertEqual(response.status_code, 200)
data = response.json()
self.assertEqual(data['ipv4_address'], '127.0.0.1')
async def get_csr_signature(self, secret: Secret, csr: CSR,
issuing_ca: CaSecret,
private_key_password: str = None) -> None:
'''
Gets the signed cert(chain) for the CSR and saves returned cert and the
existing private key.
If the issuing_ca parameter is specified then the CSR will be signed
directly by the private key of the issuing_ca, otherwise the CSR will be
send in a POST /api/v1/network/service API call to the directory server
of the network
'''
if (isinstance(secret, ServiceCaSecret) and (
self.registration_status != RegistrationStatus.Unknown
or await secret.cert_file_exists())):
# TODO: support renewal of ServiceCA cert
raise ValueError('ServiceCA cert has already been signed')
if issuing_ca:
# We have the private key of the issuing CA so can
# sign ourselves and be done with it
issuing_ca.review_csr(csr, source=CsrSource.LOCAL)
certchain = issuing_ca.sign_csr(csr)
secret.from_signed_cert(certchain)
await secret.save(password=private_key_password, overwrite=False)
# We do not set self.registration_status as locally signing
# does not provide information about the status of service in
# the network
return
if not isinstance(secret, ServiceCaSecret):
raise ValueError(
f'No issuing_ca was provided for creating a '
f'{type(secret)}'
)
# We have to get our signature from the directory server
data = {
'csr': str(
csr.public_bytes(serialization.Encoding.PEM), 'utf-8'
)
}
url = self.paths.get(Paths.NETWORKSERVICE_POST_API)
response = await RestApiClient.call(
url, HttpMethod.POST, data=data
)
if response.status != 201:
raise ValueError(
f'Failed to POST to API {Paths.NETWORKSERVICE_API}: '
f'{response.status}'
)
data = await response.json()
secret.from_string(data['signed_cert'] + data['cert_chain'])
self.registration_status = RegistrationStatus.CsrSigned
await secret.save(password=private_key_password, overwrite=False)
# Every time we receive the network data cert, we
# save it as it could have changed since the last time we
# got it
network = config.server.network
if not network.data_secret:
network.data_secret = NetworkDataSecret(network.paths)
network.data_secret.from_string(data['network_data_cert_chain'])
await network.data_secret.save(overwrite=True)
async def _create_secret(self, secret_cls: Callable, issuing_ca: Secret
) -> Secret:
'''
Abstraction for creating secrets for the Service class to avoid
repetition of code for creating the various member secrets of the
Service class
:param secret_cls: callable for one of the classes derived from
byoda.util.secrets.Secret
:raises: ValueError, NotImplementedError
'''
if not self.account_id:
raise ValueError(
'Account_id for the account has not been defined'
)
secret = secret_cls(
self.account, self.account_id, network=self.network
)
if await secret.cert_file_exists():
raise ValueError(
f'Cert for {type(secret)} for account_id {self.account_id} '
'already exists'
)
if await secret.private_key_file_exists():
raise ValueError(
f'Private key for {type(secret)} for account_id '
f'{self.account_id} already exists'
)
if not issuing_ca:
if secret_cls != AccountSecret and secret_cls != AccountDataSecret:
raise ValueError(
f'No issuing_ca was provided for creating a '
f'{type(secret_cls)}'
)
else:
csr = secret.create_csr(self.account_id)
payload = {'csr': secret.csr_as_pem(csr).decode('utf-8')}
url = self.paths.get(Paths.NETWORKACCOUNT_API)
# TODO: Refactor to use RestClientApi
_LOGGER.debug(f'Getting CSR signed from {url}')
resp = await RestApiClient.call(
url, method=HttpMethod.POST, data=payload
)
if resp.status != 201:
raise RuntimeError('Certificate signing request failed')
cert_data = await resp.json()
secret.from_string(
cert_data['signed_cert'], certchain=cert_data['cert_chain']
)
else:
csr = secret.create_csr()
issuing_ca.review_csr(csr, source=CsrSource.LOCAL)
certchain = issuing_ca.sign_csr(csr)
secret.from_signed_cert(certchain)
await secret.save(password=self.private_key_password)
return secret
async def post_service(request: Request,
csr: CertSigningRequestModel,
auth: ServiceRequestOptionalAuthFast = Depends(
ServiceRequestOptionalAuthFast),
db_session=Depends(asyncdb_session)):
'''
Submit a Certificate Signing Request for the ServiceCA certificate
and get the cert signed by the network services CA
This API is called by services
This API does not require authentication, it needs to be rate
limited by the reverse proxy
'''
_LOGGER.debug(f'POST Service API called from {request.client.host}')
network = config.server.network
dnsdb: DnsDb = config.server.network.dnsdb
# Authorization
csr_x509: x509 = Secret.csr_from_string(csr.csr)
common_name = Secret.extract_commonname(csr_x509)
try:
entity_id = NetworkServicesCaSecret.review_commonname_by_parameters(
common_name, network.name)
except PermissionError:
raise HTTPException(status_code=401,
detail=f'Invalid common name {common_name} in CSR')
except (ValueError, KeyError):
raise HTTPException(
status_code=400,
detail=(
f'error when reviewing the common name {common_name} in your '
'CSR'))
try:
await dnsdb.lookup_fqdn(common_name, DnsRecordType.A, db_session)
dns_exists = True
except KeyError:
dns_exists = False
if auth.is_authenticated:
if auth.auth_source != AuthSource.CERT:
raise HTTPException(
status_code=401,
detail=('When used with credentials, this API requires '
'authentication with TLS client cert'))
if auth.id_type != IdType.SERVICE:
raise HTTPException(
status_code=401,
detail='A TLS cert of a service is required for this API')
if auth.service_id != entity_id.service_id:
raise HTTPException(
status_code=403,
detail=(
f'Client auth for service id {auth.service_id} does not '
f'match CSR for service_id {entity_id.service_id}'))
else:
if dns_exists:
raise HTTPException(
status_code=403,
detail=(
'CSR is for existing service, must use TLS Client cert '
'for authentication'))
# End of Authorization
# The Network Services CA signs the CSRs for Service CAs
certstore = CertStore(network.services_ca)
certchain = certstore.sign(csr.csr, IdType.SERVICE_CA, request.client.host)
# Create the service and add it to the network
service = Service(network=network, service_id=entity_id.service_id)
network.services[entity_id.service_id] = service
# Get the certs as strings so we can return them
signed_cert = certchain.cert_as_string()
cert_chain = certchain.cert_chain_as_string()
# We save the public key in the network directory tree. Not sure
# if we actually need to do this as we can check any cert of the service
# and its members through the cert chain that is chained to the network
# root CA
service.service_ca = ServiceCaSecret(None, service.service_id, network)
service.service_ca.cert = certchain.signed_cert
service.service_ca.cert_chain = certchain.cert_chain
# If someone else already registered a Service then saving the cert will
# raise an exception
try:
await service.service_ca.save(overwrite=False)
except PermissionError:
raise HTTPException(409, 'Service CA certificate already exists')
# We create the DNS entry if it not exists yet to make sure there is no
# race condition between submitting the CSR through the POST API and
# registering the service server through the PUT API
if not dns_exists:
await dnsdb.create_update(None,
IdType.SERVICE,
auth.remote_addr,
db_session,
service_id=entity_id.service_id)
data_cert = network.data_secret.cert_as_pem()
return {
'signed_cert': signed_cert,
'cert_chain': cert_chain,
'network_data_cert_chain': data_cert,
}
async def post_member(request: Request,
csr: CertSigningRequestModel,
auth: MemberRequestAuthOptionalFast = Depends(
MemberRequestAuthOptionalFast)):
'''
Submit a Certificate Signing Request for the Member certificate
and get the cert signed by the Service Members CA
This API is called by pods
This API does not require authentication, it needs to be rate
limited by the reverse proxy (TODO: security)
'''
_LOGGER.debug(f'POST Member API called from {request.client.host}')
await auth.authenticate()
server: ServiceServer = config.server
service: Service = server.service
network: Network = server.network
# Authorization
csr_x509: x509 = Secret.csr_from_string(csr.csr)
common_name = Secret.extract_commonname(csr_x509)
try:
entity_id = MembersCaSecret.review_commonname_by_parameters(
common_name, network.name, service.service_id)
except PermissionError:
raise HTTPException(status_code=401,
detail=f'Invalid common name {common_name} in CSR')
except (ValueError, KeyError):
raise HTTPException(
status_code=400,
detail=(
f'error when reviewing the common name {common_name} in your '
'CSR'))
if auth.is_authenticated:
if auth.auth_source != AuthSource.CERT:
raise HTTPException(
status_code=401,
detail=('When used with credentials, this API requires '
'authentication with a TLS client cert'))
if entity_id.id_type != IdType.MEMBER:
raise HTTPException(
status_code=403,
detail='A TLS cert of a member must be used with this API')
_LOGGER.debug(f'Signing csr for existing member {entity_id.id}')
else:
# TODO: security: consider tracking member UUIDs to avoid
# race condition between CSR signature and member registration
# with the Directory server
ips = server.dns_resolver.resolve(common_name)
if ips:
_LOGGER.debug('Attempt to submit CSR for existing member without '
'authentication')
raise HTTPException(
status_code=401,
detail=(
'Must use TLS client cert when renewing a member cert'))
_LOGGER.debug(f'Signing csr for new member {entity_id.id}')
# End of Authorization
if entity_id.service_id is None:
raise ValueError(f'No service id found in common name {common_name}')
if entity_id.service_id != service.service_id:
raise HTTPException(
404, f'Incorrect service_id in common name {common_name}')
# The Network Services CA signs the CSRs for Service CAs
certstore = CertStore(service.members_ca)
certchain = certstore.sign(csr.csr, IdType.MEMBER, request.client.host)
# Get the certs as strings so we can return them
signed_cert = certchain.cert_as_string()
cert_chain = certchain.cert_chain_as_string()
service_data_cert_chain = service.data_secret.cert_as_pem()
_LOGGER.info(f'Signed certificate with commonname {common_name}')
config.server.member_db.add_meta(entity_id.id, request.client.host, None,
cert_chain, MemberStatus.SIGNED)
return {
'signed_cert': signed_cert,
'cert_chain': cert_chain,
'service_data_cert_chain': service_data_cert_chain,
}