async def run_bootstrap_tasks(data: Dict): ''' When we are bootstrapping, we create any data that is missing from the data store. ''' config.server = PodServer() server = config.server await server.set_document_store( DocumentStoreType.OBJECT_STORE, cloud_type=CloudType(data['cloud']), bucket_prefix=data['bucket_prefix'], root_dir=data['root_dir'] ) network = Network(data, data) await network.load_network_secrets() server.network = network server.paths = network.paths account = Account(data['account_id'], network) await account.paths.create_account_directory() await account.load_memberships() server.account = account _LOGGER.debug('Running bootstrap tasks') try: await account.tls_secret.load( password=account.private_key_password ) common_name = account.tls_secret.common_name if not common_name.startswith(str(account.account_id)): error_msg = ( f'Common name of existing account secret {common_name} ' 'does not match ACCOUNT_ID environment variable ' f'{data["account_id"]}' ) _LOGGER.exception(error_msg) raise ValueError(error_msg) _LOGGER.debug('Read account TLS secret') except FileNotFoundError: await account.create_account_secret() _LOGGER.info('Created account secret during bootstrap') try: await account.data_secret.load( password=account.private_key_password ) _LOGGER.debug('Read account data secret') except FileNotFoundError: await account.create_data_secret() _LOGGER.info('Created account secret during bootstrap') _LOGGER.debug('Podworker exiting normally')
async def asyncSetUp(self): try: shutil.rmtree(TEST_DIR) except FileNotFoundError: pass os.makedirs(TEST_DIR) os.environ['ROOT_DIR'] = TEST_DIR os.environ['BUCKET_PREFIX'] = 'byoda' os.environ['CLOUD'] = 'LOCAL' os.environ['NETWORK'] = 'byoda.net' os.environ['ACCOUNT_ID'] = str(get_test_uuid()) os.environ['ACCOUNT_SECRET'] = 'test' os.environ['LOGLEVEL'] = 'DEBUG' os.environ['PRIVATE_KEY_SECRET'] = 'byoda' os.environ['BOOTSTRAP'] = 'BOOTSTRAP' # Remaining environment variables used: network_data = get_environment_vars() network = Network(network_data, network_data) await network.load_network_secrets() config.server = PodServer(network) server = config.server await server.set_document_store( DocumentStoreType.OBJECT_STORE, cloud_type=CloudType(network_data['cloud']), bucket_prefix=network_data['bucket_prefix'], root_dir=network_data['root_dir']) server.paths = network.paths pod_account = Account(network_data['account_id'], network) await pod_account.paths.create_account_directory() await pod_account.load_memberships() server.account = pod_account await pod_account.create_account_secret() await pod_account.create_data_secret() await pod_account.register() await server.get_registered_services() member_id = get_test_uuid() await pod_account.join(ADDRESSBOOK_SERVICE, 1, member_id=member_id)
def get_environment_vars() -> Dict: ''' Parses environment variables. Returns dict with: - cloud: CloudType - bucket_prefix: str - network: str - account_id: str - account_secret: str - private_key_password: str - loglevel: None, 'DEBUG', 'INFO', 'WARNING', 'ERROR' or 'CRIT' - root_dir: str - roles: ['pod'] - debug: bool - bootstrap: bool - daemonize: bool, only used for podworker ''' data = { 'cloud': CloudType(os.environ.get('CLOUD', 'LOCAL')), 'bucket_prefix': os.environ.get('BUCKET_PREFIX'), 'network': os.environ.get('NETWORK', config.DEFAULT_NETWORK), 'account_id': os.environ.get('ACCOUNT_ID'), 'account_secret': os.environ.get('ACCOUNT_SECRET'), 'private_key_password': os.environ.get('PRIVATE_KEY_SECRET', 'byoda'), 'loglevel': os.environ.get('LOGLEVEL', 'WARNING'), 'root_dir': os.environ.get('ROOT_DIR'), 'daemonize': os.environ.get('DAEMONIZE', ''), 'roles': ['pod'], } if data['cloud'] == CloudType.LOCAL and not data['root_dir']: data['root_dir'] = os.environ['HOME'] + '/.byoda' data['debug'] = False if data.get('loglevel', '').upper() == 'DEBUG': data['debug'] = True data['bootstrap'] = False if os.environ.get('BOOTSTRAP', '').upper() == 'BOOTSTRAP': data['bootstrap'] = True if data.get('daemonize', '').upper() == 'FALSE': data['daemonize'] = False else: data['daemonize'] = True return data
async def get_storage(cloud: CloudType, bucket_prefix: str, root_dir: str = None): ''' Factory for FileStorage and classes derived from it For GCP, the environment variable :param cloud: the cloud that we are looking to use for object storage :param bucket: the bucket storing the data :param root_dir: the directory on the local file system that will be used to cache content :returns: instance of FileStorage or a class derived from it ''' if isinstance(cloud, str): cloud = CloudType(cloud) if cloud == CloudType.AWS: from .aws import AwsFileStorage storage = await AwsFileStorage.setup(bucket_prefix, root_dir) elif cloud == CloudType.AZURE: from .azure import AzureFileStorage storage = await AzureFileStorage.setup(bucket_prefix, root_dir) elif cloud == CloudType.GCP: from .gcp import GcpFileStorage storage = await GcpFileStorage.setup(bucket_prefix, root_dir) elif cloud == CloudType.LOCAL: _LOGGER.debug('Using LOCAL storage') storage = await FileStorage.setup(root_dir) else: raise NotImplementedError( f'There is no support for cloud {cloud}' ) _LOGGER.debug(f'Initialized {cloud} storage') return storage
async def test_secrets(self): ''' Create a network CA hierarchy ''' # # Test creation of the CA hierarchy network = await Network.create(NETWORK, TEST_DIR, 'byoda') config.server = DirectoryServer(network) config.server.network = network await config.server.set_document_store(DocumentStoreType.OBJECT_STORE, cloud_type=CloudType('LOCAL'), bucket_prefix='byoda', root_dir=TEST_DIR) network.services_ca.validate(network.root_ca, with_openssl=True) network.accounts_ca.validate(network.root_ca, with_openssl=True) # Need to set role to allow loading of unsigned services network.roles = [ServerRole.Pod] target_dir = \ f'/network-{NETWORK}/services/service-{SERVICE_ID}' os.makedirs(TEST_DIR + target_dir) target_schema = target_dir + '/service-contract.json' shutil.copy(DEFAULT_SCHEMA, TEST_DIR + target_schema) service = Service(network=network) await service.examine_servicecontract(target_schema) await service.create_secrets(network.services_ca, local=True) service.service_ca.validate(network.root_ca, with_openssl=True) service.apps_ca.validate(network.root_ca, with_openssl=True) service.tls_secret.validate(network.root_ca, with_openssl=True) service.data_secret.validate(network.root_ca, with_openssl=True) account_id = uuid4() account = Account(account_id, network) await account.paths.create_account_directory() await account.load_memberships() await account.create_secrets(network.accounts_ca) account.tls_secret.validate(network.root_ca, with_openssl=True) account.data_secret.validate(network.root_ca, with_openssl=True) # Create a dummy entry for the services in the network, otherwise # account.join(service) fails network.services = {SERVICE_ID: service} target_dir = f'/network-{NETWORK}/account-pod/service-{SERVICE_ID}' os.makedirs(TEST_DIR + target_dir) target_schema = target_dir + '/service-contract.json' shutil.copy(DEFAULT_SCHEMA, TEST_DIR + target_schema) # TODO: re-enable this test # member = account.join( # SERVICE_ID, SCHEMA_VERSION, members_ca=service.members_ca # ) # self.assertIsNotNone(member.member_id) # member.tls_secret.validate(network.root_ca, with_openssl=True) # member.data_secret.validate(network.root_ca, with_openssl=True) # Certchain validation fails as network.services_ca # is in the cert chain of account.data_secret and is # not the root CA with self.assertRaises(ValueError): account.data_secret.validate(network.services_ca) # # Test data encryption # target_account_id = uuid4() target_account = Account(target_account_id, network, account='test') await target_account.paths.create_account_directory() await target_account.load_memberships() await target_account.create_secrets(network.accounts_ca) account.data_secret.create_shared_key(target_account.data_secret) target_account.data_secret.load_shared_key( account.data_secret.protected_shared_key) self.assertEqual(account.data_secret.shared_key, target_account.data_secret.shared_key) with open('/etc/passwd', 'rb') as file_desc: data = file_desc.read() ciphertext = account.data_secret.encrypt(data) passwords = target_account.data_secret.decrypt(ciphertext) self.assertEqual(data, passwords)
async def setup(): server: PodServer = PodServer() config.server = server # Remaining environment variables used: network_data = get_environment_vars() if str(network_data['debug']).lower() == 'true': config.debug = True global _LOGGER _LOGGER = Logger.getLogger(sys.argv[0], json_out=False, debug=config.debug, loglevel=network_data['loglevel'], logfile=LOG_FILE) await server.set_document_store( DocumentStoreType.OBJECT_STORE, cloud_type=CloudType(network_data['cloud']), bucket_prefix=network_data['bucket_prefix'], root_dir=network_data['root_dir']) network = Network(network_data, network_data) await network.load_network_secrets() server.network = network server.paths = network.paths await server.get_registered_services() # TODO: if we have a pod secret, should we compare its commonname with the # account_id environment variable? pod_account = Account(network_data['account_id'], network) await pod_account.paths.create_account_directory() await pod_account.load_memberships() pod_account.password = network_data.get('account_secret') await pod_account.tls_secret.load(password=pod_account.private_key_password ) await pod_account.data_secret.load( password=pod_account.private_key_password) await pod_account.register() server.account = pod_account nginx_config = NginxConfig( directory=NGINX_SITE_CONFIG_DIR, filename='virtualserver.conf', identifier=network_data['account_id'], subdomain=IdType.ACCOUNT.value, cert_filepath=(server.paths.root_directory + '/' + pod_account.tls_secret.cert_file), key_filepath=pod_account.tls_secret.unencrypted_private_key_file, alias=network.paths.account, network=network.name, public_cloud_endpoint=network.paths.storage_driver.get_url( StorageType.PUBLIC), private_cloud_endpoint=network.paths.storage_driver.get_url( StorageType.PRIVATE), port=PodServer.HTTP_PORT, root_dir=server.network.paths.root_directory) nginx_config.create(htaccess_password=pod_account.password) nginx_config.reload() for account_member in pod_account.memberships.values(): account_member.enable_graphql_api(app) await account_member.update_registration()
async def asyncSetUp(self): config.debug = True try: shutil.rmtree(TEST_DIR) except FileNotFoundError: pass os.makedirs(TEST_DIR) shutil.copy('tests/collateral/addressbook.json', TEST_DIR) os.environ['ROOT_DIR'] = TEST_DIR os.environ['BUCKET_PREFIX'] = 'byoda' os.environ['CLOUD'] = 'LOCAL' os.environ['NETWORK'] = 'byoda.net' os.environ['ACCOUNT_ID'] = str(get_test_uuid()) os.environ['ACCOUNT_SECRET'] = 'test' os.environ['LOGLEVEL'] = 'DEBUG' os.environ['PRIVATE_KEY_SECRET'] = 'byoda' os.environ['BOOTSTRAP'] = 'BOOTSTRAP' # Remaining environment variables used: network_data = get_environment_vars() network = Network(network_data, network_data) await network.load_network_secrets() config.test_case = True config.server = PodServer(network) server = config.server global BASE_URL BASE_URL = BASE_URL.format(PORT=server.HTTP_PORT) await server.set_document_store( DocumentStoreType.OBJECT_STORE, cloud_type=CloudType(network_data['cloud']), bucket_prefix=network_data['bucket_prefix'], root_dir=network_data['root_dir']) server.paths = network.paths pod_account = Account(network_data['account_id'], network) await pod_account.paths.create_account_directory() await pod_account.load_memberships() server.account = pod_account pod_account.password = os.environ['ACCOUNT_SECRET'] await pod_account.create_account_secret() await pod_account.create_data_secret() await pod_account.register() await server.get_registered_services() service = [ service for service in server.network.service_summaries.values() if service['name'] == 'addressbook' ][0] global ADDRESSBOOK_SERVICE_ID ADDRESSBOOK_SERVICE_ID = service['service_id'] global ADDRESSBOOK_VERSION ADDRESSBOOK_VERSION = service['version'] member_id = get_test_uuid() await pod_account.join(ADDRESSBOOK_SERVICE_ID, ADDRESSBOOK_VERSION, member_id=member_id, local_service_contract='addressbook.json') app = setup_api('Byoda test pod', 'server for testing pod APIs', 'v0.0.1', [pod_account.tls_secret.common_name], [account, member, authtoken]) for account_member in pod_account.memberships.values(): account_member.enable_graphql_api(app) await account_member.update_registration() TestDirectoryApis.PROCESS = Process(target=uvicorn.run, args=(app, ), kwargs={ 'host': '0.0.0.0', 'port': config.server.HTTP_PORT, 'log_level': 'debug' }, daemon=True) TestDirectoryApis.PROCESS.start() await asyncio.sleep(2)