async def setup():
with open('config.yml') as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
debug = app_config['application']['debug']
verbose = not debug
global _LOGGER
_LOGGER = Logger.getLogger(
sys.argv[0], debug=debug, verbose=verbose,
logfile=app_config['dirserver'].get('logfile')
)
network = Network(
app_config['dirserver'], app_config['application']
)
await network.load_network_secrets()
server = DirectoryServer(network)
await server.connect_db(app_config['dirserver']['dnsdb'])
config.server = server
await server.get_registered_services()
await server.load_secrets()
if not os.environ.get('SERVER_NAME') and config.server.network.name:
os.environ['SERVER_NAME'] = config.server.network.name
async def setup():
config_file = os.environ.get('CONFIG_FILE', 'config.yml')
with open(config_file) as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
debug = app_config['application']['debug']
verbose = not debug
_LOGGER = Logger.getLogger(sys.argv[0],
debug=debug,
verbose=verbose,
logfile=app_config['svcserver'].get('logfile'))
_LOGGER.debug(f'Read configuration file: {config_file}')
network = Network(app_config['svcserver'], app_config['application'])
await network.load_network_secrets()
if not os.environ.get('SERVER_NAME') and config.server.network.name:
os.environ['SERVER_NAME'] = config.server.network.name
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
await config.server.load_secrets(
app_config['svcserver']['private_key_password'])
await config.server.load_schema()
await config.server.service.register_service()
async def asyncSetUp(self):
Logger.getLogger(sys.argv[0], debug=True, json_out=False)
with open(CONFIG_FILE) as file_desc:
TestKVCache.APP_CONFIG = yaml.load(file_desc,
Loader=yaml.SafeLoader)
app_config = TestKVCache.APP_CONFIG
test_dir = app_config['svcserver']['root_dir']
try:
shutil.rmtree(test_dir)
except FileNotFoundError:
pass
os.makedirs(test_dir)
network = await Network.create(
app_config['application']['network'],
app_config['svcserver']['root_dir'],
app_config['svcserver']['private_key_password'])
await network.load_network_secrets()
network = Network(app_config['svcserver'], app_config['application'])
await network.load_network_secrets()
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
config.server.member_db.kvcache.delete(TEST_KEY)
async def asyncSetUp(self):
Logger.getLogger(sys.argv[0], debug=True, json_out=False)
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
shutil.copy('tests/collateral/addressbook.json', TEST_DIR)
os.environ['ROOT_DIR'] = TEST_DIR
os.environ['BUCKET_PREFIX'] = 'byoda'
os.environ['CLOUD'] = 'LOCAL'
os.environ['NETWORK'] = 'byoda.net'
os.environ['ACCOUNT_ID'] = str(get_test_uuid())
os.environ['ACCOUNT_SECRET'] = 'test'
os.environ['LOGLEVEL'] = 'DEBUG'
os.environ['PRIVATE_KEY_SECRET'] = 'byoda'
os.environ['BOOTSTRAP'] = 'BOOTSTRAP'
# Remaining environment variables used:
network_data = get_environment_vars()
network = Network(network_data, network_data)
await network.load_network_secrets()
config.server = PodServer()
config.server.network = network
async def run_bootstrap_tasks(data: Dict):
'''
When we are bootstrapping, we create any data that is missing from
the data store.
'''
config.server = PodServer()
server = config.server
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(data['cloud']),
bucket_prefix=data['bucket_prefix'],
root_dir=data['root_dir']
)
network = Network(data, data)
await network.load_network_secrets()
server.network = network
server.paths = network.paths
account = Account(data['account_id'], network)
await account.paths.create_account_directory()
await account.load_memberships()
server.account = account
_LOGGER.debug('Running bootstrap tasks')
try:
await account.tls_secret.load(
password=account.private_key_password
)
common_name = account.tls_secret.common_name
if not common_name.startswith(str(account.account_id)):
error_msg = (
f'Common name of existing account secret {common_name} '
'does not match ACCOUNT_ID environment variable '
f'{data["account_id"]}'
)
_LOGGER.exception(error_msg)
raise ValueError(error_msg)
_LOGGER.debug('Read account TLS secret')
except FileNotFoundError:
await account.create_account_secret()
_LOGGER.info('Created account secret during bootstrap')
try:
await account.data_secret.load(
password=account.private_key_password
)
_LOGGER.debug('Read account data secret')
except FileNotFoundError:
await account.create_data_secret()
_LOGGER.info('Created account secret during bootstrap')
_LOGGER.debug('Podworker exiting normally')
async def load_network(args: argparse.ArgumentParser,
network_data: dict[str, str]) -> Network:
'''
Load existing network secrets
:raises: ValueError, NotImplementedError
'''
network = Network(network_data, network_data)
await network.load_network_secrets()
config.server = Server(network)
if not network.paths.network_directory_exists():
raise ValueError(f'Network {args.network} not found')
network.root_ca = NetworkRootCaSecret(network.paths)
await network.root_ca.load(with_private_key=False)
return network
async def test_network_account_post(self):
API = BASE_URL + '/v1/network/account'
network = Network(TestDirectoryApis.APP_CONFIG['dirserver'],
TestDirectoryApis.APP_CONFIG['application'])
await network.load_network_secrets()
uuid = uuid4()
secret = AccountSecret(account='dir_api_test',
account_id=uuid,
network=network)
csr = secret.create_csr()
csr = csr.public_bytes(serialization.Encoding.PEM)
fqdn = AccountSecret.create_commonname(uuid, network.name)
headers = {
'X-Client-SSL-Verify': 'SUCCESS',
'X-Client-SSL-Subject': f'CN={fqdn}',
'X-Client-SSL-Issuing-CA': f'CN=accounts-ca.{network.name}'
}
response = requests.post(API,
json={'csr': str(csr, 'utf-8')},
headers=headers)
self.assertEqual(response.status_code, 201)
data = response.json()
issuing_ca_cert = x509.load_pem_x509_certificate( # noqa:F841
data['cert_chain'].encode())
account_cert = x509.load_pem_x509_certificate( # noqa:F841
data['signed_cert'].encode())
network_data_cert = x509.load_pem_x509_certificate( # noqa:F841
data['network_data_cert_chain'].encode())
# Retry same CSR, with same TLS client cert:
response = requests.post(API,
json={'csr': str(csr, 'utf-8')},
headers=headers)
self.assertEqual(response.status_code, 201)
data = response.json()
issuing_ca_cert = x509.load_pem_x509_certificate( # noqa:F841
data['cert_chain'].encode())
account_cert = x509.load_pem_x509_certificate( # noqa:F841
data['signed_cert'].encode())
network_data_cert = x509.load_pem_x509_certificate( # noqa:F841
data['network_data_cert_chain'].encode())
# Retry same CSR, without client cert:
response = requests.post(API,
json={'csr': str(csr, 'utf-8')},
headers=None)
self.assertEqual(response.status_code, 401)
async def asyncSetUp(self):
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
os.environ['ROOT_DIR'] = TEST_DIR
os.environ['BUCKET_PREFIX'] = 'byoda'
os.environ['CLOUD'] = 'LOCAL'
os.environ['NETWORK'] = 'byoda.net'
os.environ['ACCOUNT_ID'] = str(get_test_uuid())
os.environ['ACCOUNT_SECRET'] = 'test'
os.environ['LOGLEVEL'] = 'DEBUG'
os.environ['PRIVATE_KEY_SECRET'] = 'byoda'
os.environ['BOOTSTRAP'] = 'BOOTSTRAP'
# Remaining environment variables used:
network_data = get_environment_vars()
network = Network(network_data, network_data)
await network.load_network_secrets()
config.server = PodServer(network)
server = config.server
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(network_data['cloud']),
bucket_prefix=network_data['bucket_prefix'],
root_dir=network_data['root_dir'])
server.paths = network.paths
pod_account = Account(network_data['account_id'], network)
await pod_account.paths.create_account_directory()
await pod_account.load_memberships()
server.account = pod_account
await pod_account.create_account_secret()
await pod_account.create_data_secret()
await pod_account.register()
await server.get_registered_services()
member_id = get_test_uuid()
await pod_account.join(ADDRESSBOOK_SERVICE, 1, member_id=member_id)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-d', action='store_true', default=False)
parser.add_argument('--verbose', '-v', action='store_true', default=False)
parser.add_argument('--network', '-n', type=str, default='testdomain.com')
parser.add_argument('--root-directory', '-r', type=str, default=_ROOT_DIR)
parser.add_argument('--password', '-p', type=str, default='byoda')
args = parser.parse_args()
global _LOGGER
_LOGGER = Logger.getLogger(
argv[0], debug=args.debug, verbose=args.verbose,
json_out=False
)
root_dir = args.root_directory
if root_dir.startswith('/tmp') and os.path.exists(root_dir):
_LOGGER.debug(f'Wiping temporary root directory: {root_dir}')
shutil.rmtree(root_dir)
_LOGGER.debug(
f'Creating root CA cert and private key under {args.root_directory}'
)
Network.create(args.network, root_dir, args.password)
async def main():
config_file = os.environ.get('CONFIG_FILE', 'config.yml')
with open(config_file) as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
global _LOGGER
debug = app_config['application']['debug']
_LOGGER = Logger.getLogger(
sys.argv[0],
json_out=True,
debug=app_config['application'].get('debug', False),
loglevel=app_config['application'].get('loglevel', 'INFO'),
logfile=app_config['svcserver'].get('worker_logfile'))
if debug:
global MAX_WAIT
MAX_WAIT = 10
network = Network(app_config['dirserver'], app_config['application'])
server = ServiceServer(network, app_config)
await server.load_network_secrets()
await server.load_secrets(
password=app_config['svcserver']['private_key_password'])
config.server = server
service = server.service
try:
unprotected_key_file = service.tls_secret.save_tmp_private_key()
except PermissionError:
_LOGGER.info('Could not write unprotected key, probably because it '
'already exists')
unprotected_key_file = f'/tmp/service-{service.service_id}.key'
certkey = (service.paths.root_directory + '/' +
service.tls_secret.cert_file, unprotected_key_file)
root_ca_certfile = (
f'{service.paths.root_directory}/{service.network.root_ca.cert_file}')
if not await service.paths.service_file_exists(service.service_id):
await service.download_schema(save=True)
await server.load_schema(verify_contract_signatures=False)
_LOGGER.debug(
f'Starting service worker for service ID: {service.service_id}')
while True:
member_id = server.member_db.get_next(timeout=MAX_WAIT)
if not member_id:
_LOGGER.debug('No member available in list of members')
continue
server.member_db.add_member(member_id)
_LOGGER.debug(f'Processing member_id {member_id}')
try:
data = server.member_db.get_meta(member_id)
except (TypeError, KeyError):
_LOGGER.warning(f'Invalid data for member: {member_id}')
continue
server.member_db.add_meta(data['member_id'], data['remote_addr'],
data['schema_version'], data['data_secret'],
data['status'])
waittime = next_member_wait(data['last_seen'])
#
# Here is where we can do stuff
#
url = BASE_URL.format(member_id=str(member_id),
service_id=service.service_id,
network=service.network.name)
client = GraphqlClient(endpoint=url,
cert=certkey,
verify=root_ca_certfile)
try:
result = client.execute(query=CLIENT_QUERY)
if result.get('data'):
person_data = result['data']['person']
server.member_db.set_data(member_id, person_data)
server.member_db.kvcache.set(person_data['email'],
str(member_id))
else:
_LOGGER.debug(
f'GraphQL person query failed against member {member_id}')
except (HTTPError, RequestConnectionError) as exc:
_LOGGER.debug(f'Failed to connect to {url}: {exc}')
continue
#
# and now we wait for the time to process the next client
#
asyncio.sleep(waittime)
async def setup():
server: PodServer = PodServer()
config.server = server
# Remaining environment variables used:
network_data = get_environment_vars()
if str(network_data['debug']).lower() == 'true':
config.debug = True
global _LOGGER
_LOGGER = Logger.getLogger(sys.argv[0],
json_out=False,
debug=config.debug,
loglevel=network_data['loglevel'],
logfile=LOG_FILE)
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(network_data['cloud']),
bucket_prefix=network_data['bucket_prefix'],
root_dir=network_data['root_dir'])
network = Network(network_data, network_data)
await network.load_network_secrets()
server.network = network
server.paths = network.paths
await server.get_registered_services()
# TODO: if we have a pod secret, should we compare its commonname with the
# account_id environment variable?
pod_account = Account(network_data['account_id'], network)
await pod_account.paths.create_account_directory()
await pod_account.load_memberships()
pod_account.password = network_data.get('account_secret')
await pod_account.tls_secret.load(password=pod_account.private_key_password
)
await pod_account.data_secret.load(
password=pod_account.private_key_password)
await pod_account.register()
server.account = pod_account
nginx_config = NginxConfig(
directory=NGINX_SITE_CONFIG_DIR,
filename='virtualserver.conf',
identifier=network_data['account_id'],
subdomain=IdType.ACCOUNT.value,
cert_filepath=(server.paths.root_directory + '/' +
pod_account.tls_secret.cert_file),
key_filepath=pod_account.tls_secret.unencrypted_private_key_file,
alias=network.paths.account,
network=network.name,
public_cloud_endpoint=network.paths.storage_driver.get_url(
StorageType.PUBLIC),
private_cloud_endpoint=network.paths.storage_driver.get_url(
StorageType.PRIVATE),
port=PodServer.HTTP_PORT,
root_dir=server.network.paths.root_directory)
nginx_config.create(htaccess_password=pod_account.password)
nginx_config.reload()
for account_member in pod_account.memberships.values():
account_member.enable_graphql_api(app)
await account_member.update_registration()
async def asyncSetUp(self):
config.debug = True
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
shutil.copy('tests/collateral/addressbook.json', TEST_DIR)
os.environ['ROOT_DIR'] = TEST_DIR
os.environ['BUCKET_PREFIX'] = 'byoda'
os.environ['CLOUD'] = 'LOCAL'
os.environ['NETWORK'] = 'byoda.net'
os.environ['ACCOUNT_ID'] = str(get_test_uuid())
os.environ['ACCOUNT_SECRET'] = 'test'
os.environ['LOGLEVEL'] = 'DEBUG'
os.environ['PRIVATE_KEY_SECRET'] = 'byoda'
os.environ['BOOTSTRAP'] = 'BOOTSTRAP'
# Remaining environment variables used:
network_data = get_environment_vars()
network = Network(network_data, network_data)
await network.load_network_secrets()
config.test_case = True
config.server = PodServer(network)
server = config.server
global BASE_URL
BASE_URL = BASE_URL.format(PORT=server.HTTP_PORT)
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(network_data['cloud']),
bucket_prefix=network_data['bucket_prefix'],
root_dir=network_data['root_dir'])
server.paths = network.paths
pod_account = Account(network_data['account_id'], network)
await pod_account.paths.create_account_directory()
await pod_account.load_memberships()
server.account = pod_account
pod_account.password = os.environ['ACCOUNT_SECRET']
await pod_account.create_account_secret()
await pod_account.create_data_secret()
await pod_account.register()
await server.get_registered_services()
service = [
service for service in server.network.service_summaries.values()
if service['name'] == 'addressbook'
][0]
global ADDRESSBOOK_SERVICE_ID
ADDRESSBOOK_SERVICE_ID = service['service_id']
global ADDRESSBOOK_VERSION
ADDRESSBOOK_VERSION = service['version']
member_id = get_test_uuid()
await pod_account.join(ADDRESSBOOK_SERVICE_ID,
ADDRESSBOOK_VERSION,
member_id=member_id,
local_service_contract='addressbook.json')
app = setup_api('Byoda test pod', 'server for testing pod APIs',
'v0.0.1', [pod_account.tls_secret.common_name],
[account, member, authtoken])
for account_member in pod_account.memberships.values():
account_member.enable_graphql_api(app)
await account_member.update_registration()
TestDirectoryApis.PROCESS = Process(target=uvicorn.run,
args=(app, ),
kwargs={
'host': '0.0.0.0',
'port':
config.server.HTTP_PORT,
'log_level': 'debug'
},
daemon=True)
TestDirectoryApis.PROCESS.start()
await asyncio.sleep(2)
async def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-d', action='store_true', default=False)
parser.add_argument('--verbose', '-v', action='store_true', default=False)
parser.add_argument(
'--signing-party', '-s', type=str, default=None,
choices=[i.value for i in SignatureType]
)
parser.add_argument('--contract', '-c', type=str)
parser.add_argument('--config', '-o', type=str, default='config.yml')
parser.add_argument('--local', default=False, action='store_true')
args = parser.parse_args()
# Network constructor expects parameters to be in a dict
with open(args.config) as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
root_dir = app_config['svcserver']['root_dir']
password = app_config['svcserver']['private_key_password']
global _LOGGER
_LOGGER = Logger.getLogger(
argv[0], debug=args.debug, verbose=args.verbose,
json_out=False
)
network = Network(
app_config['svcserver'], app_config['application']
)
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
service = await load_service(args, config.server.network, password)
if not args.local:
service.registration_status = await service.get_registration_status()
if service.registration_status == RegistrationStatus.Unknown:
raise ValueError(
'Please use "create_service_secrets.py" script first'
)
schema = service.schema.json_schema
if 'signatures' not in schema:
schema['signatures'] = {}
if not args.local:
response = await service.register_service()
if response.status != 200:
raise ValueError(
f'Failed to register service: {response.status}'
)
result = None
if (not args.signing_party
or args.signing_party == SignatureType.SERVICE.value):
try:
create_service_signature(service)
result = True
except Exception as exc:
_LOGGER.exception(f'Failed to get service signature: {exc}')
result = False
if (result is not False and (
not args.signing_party
or args.signing_party == SignatureType.NETWORK.value)):
result = await create_network_signature(service, args, password)
if not result:
_LOGGER.error('Failed to get the network signature')
sys.exit(1)
if not root_dir.startswith('/'):
# Hack to make relative paths work with the FileStorage class
root_dir = os.getcwd()
storage_driver = FileStorage(root_dir)
filepath = service.paths.get(Paths.SERVICE_FILE)
_LOGGER.debug(f'Saving signed schema to {filepath}')
await service.schema.save(filepath, storage_driver=storage_driver)