async def run_bootstrap_tasks(data: Dict):
'''
When we are bootstrapping, we create any data that is missing from
the data store.
'''
config.server = PodServer()
server = config.server
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(data['cloud']),
bucket_prefix=data['bucket_prefix'],
root_dir=data['root_dir']
)
network = Network(data, data)
await network.load_network_secrets()
server.network = network
server.paths = network.paths
account = Account(data['account_id'], network)
await account.paths.create_account_directory()
await account.load_memberships()
server.account = account
_LOGGER.debug('Running bootstrap tasks')
try:
await account.tls_secret.load(
password=account.private_key_password
)
common_name = account.tls_secret.common_name
if not common_name.startswith(str(account.account_id)):
error_msg = (
f'Common name of existing account secret {common_name} '
'does not match ACCOUNT_ID environment variable '
f'{data["account_id"]}'
)
_LOGGER.exception(error_msg)
raise ValueError(error_msg)
_LOGGER.debug('Read account TLS secret')
except FileNotFoundError:
await account.create_account_secret()
_LOGGER.info('Created account secret during bootstrap')
try:
await account.data_secret.load(
password=account.private_key_password
)
_LOGGER.debug('Read account data secret')
except FileNotFoundError:
await account.create_data_secret()
_LOGGER.info('Created account secret during bootstrap')
_LOGGER.debug('Podworker exiting normally')
async def asyncSetUp(self):
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
os.environ['ROOT_DIR'] = TEST_DIR
os.environ['BUCKET_PREFIX'] = 'byoda'
os.environ['CLOUD'] = 'LOCAL'
os.environ['NETWORK'] = 'byoda.net'
os.environ['ACCOUNT_ID'] = str(get_test_uuid())
os.environ['ACCOUNT_SECRET'] = 'test'
os.environ['LOGLEVEL'] = 'DEBUG'
os.environ['PRIVATE_KEY_SECRET'] = 'byoda'
os.environ['BOOTSTRAP'] = 'BOOTSTRAP'
# Remaining environment variables used:
network_data = get_environment_vars()
network = Network(network_data, network_data)
await network.load_network_secrets()
config.server = PodServer(network)
server = config.server
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(network_data['cloud']),
bucket_prefix=network_data['bucket_prefix'],
root_dir=network_data['root_dir'])
server.paths = network.paths
pod_account = Account(network_data['account_id'], network)
await pod_account.paths.create_account_directory()
await pod_account.load_memberships()
server.account = pod_account
await pod_account.create_account_secret()
await pod_account.create_data_secret()
await pod_account.register()
await server.get_registered_services()
member_id = get_test_uuid()
await pod_account.join(ADDRESSBOOK_SERVICE, 1, member_id=member_id)
def get_environment_vars() -> Dict:
'''
Parses environment variables. Returns dict with:
- cloud: CloudType
- bucket_prefix: str
- network: str
- account_id: str
- account_secret: str
- private_key_password: str
- loglevel: None, 'DEBUG', 'INFO', 'WARNING', 'ERROR' or 'CRIT'
- root_dir: str
- roles: ['pod']
- debug: bool
- bootstrap: bool
- daemonize: bool, only used for podworker
'''
data = {
'cloud': CloudType(os.environ.get('CLOUD', 'LOCAL')),
'bucket_prefix': os.environ.get('BUCKET_PREFIX'),
'network': os.environ.get('NETWORK', config.DEFAULT_NETWORK),
'account_id': os.environ.get('ACCOUNT_ID'),
'account_secret': os.environ.get('ACCOUNT_SECRET'),
'private_key_password': os.environ.get('PRIVATE_KEY_SECRET', 'byoda'),
'loglevel': os.environ.get('LOGLEVEL', 'WARNING'),
'root_dir': os.environ.get('ROOT_DIR'),
'daemonize': os.environ.get('DAEMONIZE', ''),
'roles': ['pod'],
}
if data['cloud'] == CloudType.LOCAL and not data['root_dir']:
data['root_dir'] = os.environ['HOME'] + '/.byoda'
data['debug'] = False
if data.get('loglevel', '').upper() == 'DEBUG':
data['debug'] = True
data['bootstrap'] = False
if os.environ.get('BOOTSTRAP', '').upper() == 'BOOTSTRAP':
data['bootstrap'] = True
if data.get('daemonize', '').upper() == 'FALSE':
data['daemonize'] = False
else:
data['daemonize'] = True
return data
async def get_storage(cloud: CloudType, bucket_prefix: str,
root_dir: str = None):
'''
Factory for FileStorage and classes derived from it
For GCP, the environment variable
:param cloud: the cloud that we are looking to use for object
storage
:param bucket: the bucket storing the data
:param root_dir: the directory on the local file system that
will be used to cache content
:returns: instance of FileStorage or a class derived from it
'''
if isinstance(cloud, str):
cloud = CloudType(cloud)
if cloud == CloudType.AWS:
from .aws import AwsFileStorage
storage = await AwsFileStorage.setup(bucket_prefix, root_dir)
elif cloud == CloudType.AZURE:
from .azure import AzureFileStorage
storage = await AzureFileStorage.setup(bucket_prefix, root_dir)
elif cloud == CloudType.GCP:
from .gcp import GcpFileStorage
storage = await GcpFileStorage.setup(bucket_prefix, root_dir)
elif cloud == CloudType.LOCAL:
_LOGGER.debug('Using LOCAL storage')
storage = await FileStorage.setup(root_dir)
else:
raise NotImplementedError(
f'There is no support for cloud {cloud}'
)
_LOGGER.debug(f'Initialized {cloud} storage')
return storage
async def test_secrets(self):
'''
Create a network CA hierarchy
'''
#
# Test creation of the CA hierarchy
network = await Network.create(NETWORK, TEST_DIR, 'byoda')
config.server = DirectoryServer(network)
config.server.network = network
await config.server.set_document_store(DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType('LOCAL'),
bucket_prefix='byoda',
root_dir=TEST_DIR)
network.services_ca.validate(network.root_ca, with_openssl=True)
network.accounts_ca.validate(network.root_ca, with_openssl=True)
# Need to set role to allow loading of unsigned services
network.roles = [ServerRole.Pod]
target_dir = \
f'/network-{NETWORK}/services/service-{SERVICE_ID}'
os.makedirs(TEST_DIR + target_dir)
target_schema = target_dir + '/service-contract.json'
shutil.copy(DEFAULT_SCHEMA, TEST_DIR + target_schema)
service = Service(network=network)
await service.examine_servicecontract(target_schema)
await service.create_secrets(network.services_ca, local=True)
service.service_ca.validate(network.root_ca, with_openssl=True)
service.apps_ca.validate(network.root_ca, with_openssl=True)
service.tls_secret.validate(network.root_ca, with_openssl=True)
service.data_secret.validate(network.root_ca, with_openssl=True)
account_id = uuid4()
account = Account(account_id, network)
await account.paths.create_account_directory()
await account.load_memberships()
await account.create_secrets(network.accounts_ca)
account.tls_secret.validate(network.root_ca, with_openssl=True)
account.data_secret.validate(network.root_ca, with_openssl=True)
# Create a dummy entry for the services in the network, otherwise
# account.join(service) fails
network.services = {SERVICE_ID: service}
target_dir = f'/network-{NETWORK}/account-pod/service-{SERVICE_ID}'
os.makedirs(TEST_DIR + target_dir)
target_schema = target_dir + '/service-contract.json'
shutil.copy(DEFAULT_SCHEMA, TEST_DIR + target_schema)
# TODO: re-enable this test
# member = account.join(
# SERVICE_ID, SCHEMA_VERSION, members_ca=service.members_ca
# )
# self.assertIsNotNone(member.member_id)
# member.tls_secret.validate(network.root_ca, with_openssl=True)
# member.data_secret.validate(network.root_ca, with_openssl=True)
# Certchain validation fails as network.services_ca
# is in the cert chain of account.data_secret and is
# not the root CA
with self.assertRaises(ValueError):
account.data_secret.validate(network.services_ca)
#
# Test data encryption
#
target_account_id = uuid4()
target_account = Account(target_account_id, network, account='test')
await target_account.paths.create_account_directory()
await target_account.load_memberships()
await target_account.create_secrets(network.accounts_ca)
account.data_secret.create_shared_key(target_account.data_secret)
target_account.data_secret.load_shared_key(
account.data_secret.protected_shared_key)
self.assertEqual(account.data_secret.shared_key,
target_account.data_secret.shared_key)
with open('/etc/passwd', 'rb') as file_desc:
data = file_desc.read()
ciphertext = account.data_secret.encrypt(data)
passwords = target_account.data_secret.decrypt(ciphertext)
self.assertEqual(data, passwords)
async def setup():
server: PodServer = PodServer()
config.server = server
# Remaining environment variables used:
network_data = get_environment_vars()
if str(network_data['debug']).lower() == 'true':
config.debug = True
global _LOGGER
_LOGGER = Logger.getLogger(sys.argv[0],
json_out=False,
debug=config.debug,
loglevel=network_data['loglevel'],
logfile=LOG_FILE)
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(network_data['cloud']),
bucket_prefix=network_data['bucket_prefix'],
root_dir=network_data['root_dir'])
network = Network(network_data, network_data)
await network.load_network_secrets()
server.network = network
server.paths = network.paths
await server.get_registered_services()
# TODO: if we have a pod secret, should we compare its commonname with the
# account_id environment variable?
pod_account = Account(network_data['account_id'], network)
await pod_account.paths.create_account_directory()
await pod_account.load_memberships()
pod_account.password = network_data.get('account_secret')
await pod_account.tls_secret.load(password=pod_account.private_key_password
)
await pod_account.data_secret.load(
password=pod_account.private_key_password)
await pod_account.register()
server.account = pod_account
nginx_config = NginxConfig(
directory=NGINX_SITE_CONFIG_DIR,
filename='virtualserver.conf',
identifier=network_data['account_id'],
subdomain=IdType.ACCOUNT.value,
cert_filepath=(server.paths.root_directory + '/' +
pod_account.tls_secret.cert_file),
key_filepath=pod_account.tls_secret.unencrypted_private_key_file,
alias=network.paths.account,
network=network.name,
public_cloud_endpoint=network.paths.storage_driver.get_url(
StorageType.PUBLIC),
private_cloud_endpoint=network.paths.storage_driver.get_url(
StorageType.PRIVATE),
port=PodServer.HTTP_PORT,
root_dir=server.network.paths.root_directory)
nginx_config.create(htaccess_password=pod_account.password)
nginx_config.reload()
for account_member in pod_account.memberships.values():
account_member.enable_graphql_api(app)
await account_member.update_registration()
async def asyncSetUp(self):
config.debug = True
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
shutil.copy('tests/collateral/addressbook.json', TEST_DIR)
os.environ['ROOT_DIR'] = TEST_DIR
os.environ['BUCKET_PREFIX'] = 'byoda'
os.environ['CLOUD'] = 'LOCAL'
os.environ['NETWORK'] = 'byoda.net'
os.environ['ACCOUNT_ID'] = str(get_test_uuid())
os.environ['ACCOUNT_SECRET'] = 'test'
os.environ['LOGLEVEL'] = 'DEBUG'
os.environ['PRIVATE_KEY_SECRET'] = 'byoda'
os.environ['BOOTSTRAP'] = 'BOOTSTRAP'
# Remaining environment variables used:
network_data = get_environment_vars()
network = Network(network_data, network_data)
await network.load_network_secrets()
config.test_case = True
config.server = PodServer(network)
server = config.server
global BASE_URL
BASE_URL = BASE_URL.format(PORT=server.HTTP_PORT)
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(network_data['cloud']),
bucket_prefix=network_data['bucket_prefix'],
root_dir=network_data['root_dir'])
server.paths = network.paths
pod_account = Account(network_data['account_id'], network)
await pod_account.paths.create_account_directory()
await pod_account.load_memberships()
server.account = pod_account
pod_account.password = os.environ['ACCOUNT_SECRET']
await pod_account.create_account_secret()
await pod_account.create_data_secret()
await pod_account.register()
await server.get_registered_services()
service = [
service for service in server.network.service_summaries.values()
if service['name'] == 'addressbook'
][0]
global ADDRESSBOOK_SERVICE_ID
ADDRESSBOOK_SERVICE_ID = service['service_id']
global ADDRESSBOOK_VERSION
ADDRESSBOOK_VERSION = service['version']
member_id = get_test_uuid()
await pod_account.join(ADDRESSBOOK_SERVICE_ID,
ADDRESSBOOK_VERSION,
member_id=member_id,
local_service_contract='addressbook.json')
app = setup_api('Byoda test pod', 'server for testing pod APIs',
'v0.0.1', [pod_account.tls_secret.common_name],
[account, member, authtoken])
for account_member in pod_account.memberships.values():
account_member.enable_graphql_api(app)
await account_member.update_registration()
TestDirectoryApis.PROCESS = Process(target=uvicorn.run,
args=(app, ),
kwargs={
'host': '0.0.0.0',
'port':
config.server.HTTP_PORT,
'log_level': 'debug'
},
daemon=True)
TestDirectoryApis.PROCESS.start()
await asyncio.sleep(2)