async def main(args):
# Remaining environment variables used:
data = get_environment_vars()
global _LOGGER
_LOGGER = Logger.getLogger(
sys.argv[0], json_out=False, debug=True,
loglevel='DEBUG'
)
_LOGGER.debug(f'Starting podworker {data["bootstrap"]}')
if data['bootstrap']:
await run_bootstrap_tasks(data)
if data['daemonize']:
with daemon.DaemonContext():
_LOGGER = Logger.getLogger(
sys.argv[0], json_out=False, debug=data['debug'],
loglevel=os.environ.get('loglevel', 'ERROR'),
logfile=LOG_FILE
)
_LOGGER.debug('Daemonizing podworker')
schedule_periodic_tasks()
run_tasks()
async def asyncSetUp(self):
Logger.getLogger(sys.argv[0], debug=True, json_out=False)
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
shutil.copy('tests/collateral/addressbook.json', TEST_DIR)
os.environ['ROOT_DIR'] = TEST_DIR
os.environ['BUCKET_PREFIX'] = 'byoda'
os.environ['CLOUD'] = 'LOCAL'
os.environ['NETWORK'] = 'byoda.net'
os.environ['ACCOUNT_ID'] = str(get_test_uuid())
os.environ['ACCOUNT_SECRET'] = 'test'
os.environ['LOGLEVEL'] = 'DEBUG'
os.environ['PRIVATE_KEY_SECRET'] = 'byoda'
os.environ['BOOTSTRAP'] = 'BOOTSTRAP'
# Remaining environment variables used:
network_data = get_environment_vars()
network = Network(network_data, network_data)
await network.load_network_secrets()
config.server = PodServer()
config.server.network = network
async def asyncSetUp(self):
Logger.getLogger(sys.argv[0], debug=True, json_out=False)
with open(CONFIG_FILE) as file_desc:
TestKVCache.APP_CONFIG = yaml.load(file_desc,
Loader=yaml.SafeLoader)
app_config = TestKVCache.APP_CONFIG
test_dir = app_config['svcserver']['root_dir']
try:
shutil.rmtree(test_dir)
except FileNotFoundError:
pass
os.makedirs(test_dir)
# Create the network so that the constructor of ServiceServer
# can load it.
network: Network = await Network.create(
app_config['application']['network'],
app_config['svcserver']['root_dir'],
app_config['svcserver']['private_key_password'])
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
member_db = config.server.member_db
member_db.service_id = app_config['svcserver']['service_id']
member_db.delete_meta(TEST_MEMBER_UUID)
member_db.delete_members_list()
async def asyncSetUp(self):
Logger.getLogger(sys.argv[0], debug=True, json_out=False)
with open(CONFIG_FILE) as file_desc:
TestKVCache.APP_CONFIG = yaml.load(file_desc,
Loader=yaml.SafeLoader)
app_config = TestKVCache.APP_CONFIG
test_dir = app_config['svcserver']['root_dir']
try:
shutil.rmtree(test_dir)
except FileNotFoundError:
pass
os.makedirs(test_dir)
network = await Network.create(
app_config['application']['network'],
app_config['svcserver']['root_dir'],
app_config['svcserver']['private_key_password'])
await network.load_network_secrets()
network = Network(app_config['svcserver'], app_config['application'])
await network.load_network_secrets()
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
config.server.member_db.kvcache.delete(TEST_KEY)
async def setup():
config_file = os.environ.get('CONFIG_FILE', 'config.yml')
with open(config_file) as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
debug = app_config['application']['debug']
verbose = not debug
_LOGGER = Logger.getLogger(sys.argv[0],
debug=debug,
verbose=verbose,
logfile=app_config['svcserver'].get('logfile'))
_LOGGER.debug(f'Read configuration file: {config_file}')
network = Network(app_config['svcserver'], app_config['application'])
await network.load_network_secrets()
if not os.environ.get('SERVER_NAME') and config.server.network.name:
os.environ['SERVER_NAME'] = config.server.network.name
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
await config.server.load_secrets(
app_config['svcserver']['private_key_password'])
await config.server.load_schema()
await config.server.service.register_service()
async def setup():
with open('config.yml') as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
debug = app_config['application']['debug']
verbose = not debug
global _LOGGER
_LOGGER = Logger.getLogger(
sys.argv[0], debug=debug, verbose=verbose,
logfile=app_config['dirserver'].get('logfile')
)
network = Network(
app_config['dirserver'], app_config['application']
)
await network.load_network_secrets()
server = DirectoryServer(network)
await server.connect_db(app_config['dirserver']['dnsdb'])
config.server = server
await server.get_registered_services()
await server.load_secrets()
if not os.environ.get('SERVER_NAME') and config.server.network.name:
os.environ['SERVER_NAME'] = config.server.network.name
async def asyncSetUp(self):
Logger.getLogger(sys.argv[0], debug=True, json_out=False)
with open(CONFIG_FILE) as file_desc:
TestDirectoryApis.APP_CONFIG = yaml.load(file_desc,
Loader=yaml.SafeLoader)
app_config = TestDirectoryApis.APP_CONFIG
app_config['dirserver']['root_dir'] = TEST_DIR
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
os.makedirs(
f'{SERVICE_DIR}/network-{app_config["application"]["network"]}'
f'/services/service-{SERVICE_ID}')
network = await Network.create(
app_config['application']['network'],
app_config['dirserver']['root_dir'],
app_config['dirserver']['private_key_password'],
)
config.server = DirectoryServer(network)
await config.server.connect_db(app_config['dirserver']['dnsdb'])
app = setup_api('Byoda test dirserver',
'server for testing directory APIs', 'v0.0.1', [],
[account, service, member])
TestDirectoryApis.PROCESS = Process(target=uvicorn.run,
args=(app, ),
kwargs={
'host': '127.0.0.1',
'port': TEST_PORT,
'log_level': 'debug'
},
daemon=True)
TestDirectoryApis.PROCESS.start()
await asyncio.sleep(1)
async def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-d', action='store_true', default=False)
parser.add_argument('--verbose', '-v', action='store_true', default=False)
parser.add_argument('--schema', '-s', type=str)
parser.add_argument('--network', '-n', type=str, default='test')
parser.add_argument('--root-directory', '-r', type=str, default=_ROOT_DIR)
parser.add_argument('--password', '-p', type=str, default='byoda')
parser.add_argument('--local', default=True, action='store_false')
args = parser.parse_args()
# Network constructor expects parameters to be in a dict
network_data = {
'private_key_password': args.password,
'cloud': 'LOCAL',
'bucket_prefix': None,
'roles': [],
'network': args.network,
'root_dir': args.root_directory,
}
global _LOGGER
_LOGGER = Logger.getLogger(argv[0],
debug=args.debug,
verbose=args.verbose,
json_out=False)
root_dir = args.root_directory
if root_dir.startswith('/tmp') and os.path.exists(root_dir):
_LOGGER.debug(f'Wiping temporary root directory: {root_dir}')
shutil.rmtree(root_dir)
network_dir = f'{root_dir}/network-{args.network}'
network_cert_filepath = (network_dir +
f'/network-{args.network}-root-ca-cert.pem')
if not os.path.exists(network_cert_filepath):
os.makedirs(network_dir, exist_ok=True)
resp = requests.get(f'https://dir.{args.network}/root-ca.pem')
with open(network_cert_filepath, 'w') as file_desc:
file_desc.write(await resp.text())
network = load_network(args, network_data)
service = Service(network=network)
if args.schema:
await service.examine_servicecontract(args.schema)
await service.create_secrets(network.services_ca,
password=args.password,
local=False)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-d', action='store_true', default=False)
parser.add_argument('--verbose', '-v', action='store_true', default=False)
parser.add_argument('--network', '-n', type=str, default='testdomain.com')
parser.add_argument('--root-directory', '-r', type=str, default=_ROOT_DIR)
parser.add_argument('--password', '-p', type=str, default='byoda')
args = parser.parse_args()
global _LOGGER
_LOGGER = Logger.getLogger(
argv[0], debug=args.debug, verbose=args.verbose,
json_out=False
)
root_dir = args.root_directory
if root_dir.startswith('/tmp') and os.path.exists(root_dir):
_LOGGER.debug(f'Wiping temporary root directory: {root_dir}')
shutil.rmtree(root_dir)
_LOGGER.debug(
f'Creating root CA cert and private key under {args.root_directory}'
)
Network.create(args.network, root_dir, args.password)
test.assertFalse(exists)
subdirs = await storage.get_folders('test/')
test.assertEqual(len(subdirs), 2)
subdirs = await storage.get_folders('test/', prefix='sub')
test.assertEqual(len(subdirs), 1)
if (type(storage) in (AzureFileStorage, AwsFileStorage, GcpFileStorage)):
url = storage.get_url() + 'test/profile'
response = requests.get(url, allow_redirects=False)
test.assertIn(response.status_code, (302, 403, 409))
await storage.delete('test/profile')
await storage.delete('test/anothersubdir/profile-write')
await storage.delete('test/subdir/profile-write')
# GCP delete also deletes empty parent 'folders'??
if not type(storage) in (AzureFileStorage, GcpFileStorage):
await storage.delete('test/subdir/')
await storage.delete('test/anothersubdir/')
await storage.delete('test')
await storage.close_clients()
if __name__ == '__main__':
_LOGGER = Logger.getLogger(sys.argv[0], debug=True, json_out=False)
unittest.main()
async def main():
config_file = os.environ.get('CONFIG_FILE', 'config.yml')
with open(config_file) as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
global _LOGGER
debug = app_config['application']['debug']
_LOGGER = Logger.getLogger(
sys.argv[0],
json_out=True,
debug=app_config['application'].get('debug', False),
loglevel=app_config['application'].get('loglevel', 'INFO'),
logfile=app_config['svcserver'].get('worker_logfile'))
if debug:
global MAX_WAIT
MAX_WAIT = 10
network = Network(app_config['dirserver'], app_config['application'])
server = ServiceServer(network, app_config)
await server.load_network_secrets()
await server.load_secrets(
password=app_config['svcserver']['private_key_password'])
config.server = server
service = server.service
try:
unprotected_key_file = service.tls_secret.save_tmp_private_key()
except PermissionError:
_LOGGER.info('Could not write unprotected key, probably because it '
'already exists')
unprotected_key_file = f'/tmp/service-{service.service_id}.key'
certkey = (service.paths.root_directory + '/' +
service.tls_secret.cert_file, unprotected_key_file)
root_ca_certfile = (
f'{service.paths.root_directory}/{service.network.root_ca.cert_file}')
if not await service.paths.service_file_exists(service.service_id):
await service.download_schema(save=True)
await server.load_schema(verify_contract_signatures=False)
_LOGGER.debug(
f'Starting service worker for service ID: {service.service_id}')
while True:
member_id = server.member_db.get_next(timeout=MAX_WAIT)
if not member_id:
_LOGGER.debug('No member available in list of members')
continue
server.member_db.add_member(member_id)
_LOGGER.debug(f'Processing member_id {member_id}')
try:
data = server.member_db.get_meta(member_id)
except (TypeError, KeyError):
_LOGGER.warning(f'Invalid data for member: {member_id}')
continue
server.member_db.add_meta(data['member_id'], data['remote_addr'],
data['schema_version'], data['data_secret'],
data['status'])
waittime = next_member_wait(data['last_seen'])
#
# Here is where we can do stuff
#
url = BASE_URL.format(member_id=str(member_id),
service_id=service.service_id,
network=service.network.name)
client = GraphqlClient(endpoint=url,
cert=certkey,
verify=root_ca_certfile)
try:
result = client.execute(query=CLIENT_QUERY)
if result.get('data'):
person_data = result['data']['person']
server.member_db.set_data(member_id, person_data)
server.member_db.kvcache.set(person_data['email'],
str(member_id))
else:
_LOGGER.debug(
f'GraphQL person query failed against member {member_id}')
except (HTTPError, RequestConnectionError) as exc:
_LOGGER.debug(f'Failed to connect to {url}: {exc}')
continue
#
# and now we wait for the time to process the next client
#
asyncio.sleep(waittime)
async def setup():
server: PodServer = PodServer()
config.server = server
# Remaining environment variables used:
network_data = get_environment_vars()
if str(network_data['debug']).lower() == 'true':
config.debug = True
global _LOGGER
_LOGGER = Logger.getLogger(sys.argv[0],
json_out=False,
debug=config.debug,
loglevel=network_data['loglevel'],
logfile=LOG_FILE)
await server.set_document_store(
DocumentStoreType.OBJECT_STORE,
cloud_type=CloudType(network_data['cloud']),
bucket_prefix=network_data['bucket_prefix'],
root_dir=network_data['root_dir'])
network = Network(network_data, network_data)
await network.load_network_secrets()
server.network = network
server.paths = network.paths
await server.get_registered_services()
# TODO: if we have a pod secret, should we compare its commonname with the
# account_id environment variable?
pod_account = Account(network_data['account_id'], network)
await pod_account.paths.create_account_directory()
await pod_account.load_memberships()
pod_account.password = network_data.get('account_secret')
await pod_account.tls_secret.load(password=pod_account.private_key_password
)
await pod_account.data_secret.load(
password=pod_account.private_key_password)
await pod_account.register()
server.account = pod_account
nginx_config = NginxConfig(
directory=NGINX_SITE_CONFIG_DIR,
filename='virtualserver.conf',
identifier=network_data['account_id'],
subdomain=IdType.ACCOUNT.value,
cert_filepath=(server.paths.root_directory + '/' +
pod_account.tls_secret.cert_file),
key_filepath=pod_account.tls_secret.unencrypted_private_key_file,
alias=network.paths.account,
network=network.name,
public_cloud_endpoint=network.paths.storage_driver.get_url(
StorageType.PUBLIC),
private_cloud_endpoint=network.paths.storage_driver.get_url(
StorageType.PRIVATE),
port=PodServer.HTTP_PORT,
root_dir=server.network.paths.root_directory)
nginx_config.create(htaccess_password=pod_account.password)
nginx_config.reload()
for account_member in pod_account.memberships.values():
account_member.enable_graphql_api(app)
await account_member.update_registration()
async def asyncSetUp(self):
Logger.getLogger(sys.argv[0], debug=True, json_out=False)
with open(CONFIG_FILE) as file_desc:
TestDirectoryApis.APP_CONFIG = yaml.load(file_desc,
Loader=yaml.SafeLoader)
app_config = TestDirectoryApis.APP_CONFIG
app_config['svcserver']['service_id'] = SERVICE_ID
app_config['svcserver']['root_dir'] = TEST_DIR
try:
shutil.rmtree(TEST_DIR)
except FileNotFoundError:
pass
os.makedirs(TEST_DIR)
service_dir = (f'{TEST_DIR}/network-'
f'{app_config["application"]["network"]}'
f'/services/service-{SERVICE_ID}')
os.makedirs(service_dir)
network = await Network.create(
app_config['application']['network'], TEST_DIR,
app_config['svcserver']['private_key_password'])
service_file = network.paths.get(Paths.SERVICE_FILE,
service_id=SERVICE_ID)
shutil.copy(DUMMY_SCHEMA, TEST_DIR + '/' + service_file)
svc = Service(network=network,
service_id=app_config['svcserver']['service_id'])
if service_file:
await svc.examine_servicecontract(service_file)
await svc.create_secrets(
network.services_ca,
local=True,
password=app_config['svcserver']['private_key_password'])
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
await config.server.load_secrets(
app_config['svcserver']['private_key_password'])
await config.server.load_schema(verify_contract_signatures=False)
app = setup_api('Byoda test svcserver',
'server for testing service APIs', 'v0.0.1', [],
[service, member])
TestDirectoryApis.PROCESS = Process(target=uvicorn.run,
args=(app, ),
kwargs={
'host': '127.0.0.1',
'port': TEST_PORT,
'log_level': 'debug'
},
daemon=True)
TestDirectoryApis.PROCESS.start()
await asyncio.sleep(1)
async def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-d', action='store_true', default=False)
parser.add_argument('--verbose', '-v', action='store_true', default=False)
parser.add_argument(
'--signing-party', '-s', type=str, default=None,
choices=[i.value for i in SignatureType]
)
parser.add_argument('--contract', '-c', type=str)
parser.add_argument('--config', '-o', type=str, default='config.yml')
parser.add_argument('--local', default=False, action='store_true')
args = parser.parse_args()
# Network constructor expects parameters to be in a dict
with open(args.config) as file_desc:
app_config = yaml.load(file_desc, Loader=yaml.SafeLoader)
root_dir = app_config['svcserver']['root_dir']
password = app_config['svcserver']['private_key_password']
global _LOGGER
_LOGGER = Logger.getLogger(
argv[0], debug=args.debug, verbose=args.verbose,
json_out=False
)
network = Network(
app_config['svcserver'], app_config['application']
)
config.server = ServiceServer(network, app_config)
await config.server.load_network_secrets()
service = await load_service(args, config.server.network, password)
if not args.local:
service.registration_status = await service.get_registration_status()
if service.registration_status == RegistrationStatus.Unknown:
raise ValueError(
'Please use "create_service_secrets.py" script first'
)
schema = service.schema.json_schema
if 'signatures' not in schema:
schema['signatures'] = {}
if not args.local:
response = await service.register_service()
if response.status != 200:
raise ValueError(
f'Failed to register service: {response.status}'
)
result = None
if (not args.signing_party
or args.signing_party == SignatureType.SERVICE.value):
try:
create_service_signature(service)
result = True
except Exception as exc:
_LOGGER.exception(f'Failed to get service signature: {exc}')
result = False
if (result is not False and (
not args.signing_party
or args.signing_party == SignatureType.NETWORK.value)):
result = await create_network_signature(service, args, password)
if not result:
_LOGGER.error('Failed to get the network signature')
sys.exit(1)
if not root_dir.startswith('/'):
# Hack to make relative paths work with the FileStorage class
root_dir = os.getcwd()
storage_driver = FileStorage(root_dir)
filepath = service.paths.get(Paths.SERVICE_FILE)
_LOGGER.debug(f'Saving signed schema to {filepath}')
await service.schema.save(filepath, storage_driver=storage_driver)
