async def main(args): # Remaining environment variables used: data = get_environment_vars() global _LOGGER _LOGGER = Logger.getLogger( sys.argv[0], json_out=False, debug=True, loglevel='DEBUG' ) _LOGGER.debug(f'Starting podworker {data["bootstrap"]}') if data['bootstrap']: await run_bootstrap_tasks(data) if data['daemonize']: with daemon.DaemonContext(): _LOGGER = Logger.getLogger( sys.argv[0], json_out=False, debug=data['debug'], loglevel=os.environ.get('loglevel', 'ERROR'), logfile=LOG_FILE ) _LOGGER.debug('Daemonizing podworker') schedule_periodic_tasks() run_tasks()
async def asyncSetUp(self): Logger.getLogger(sys.argv[0], debug=True, json_out=False) try: shutil.rmtree(TEST_DIR) except FileNotFoundError: pass os.makedirs(TEST_DIR) shutil.copy('tests/collateral/addressbook.json', TEST_DIR) os.environ['ROOT_DIR'] = TEST_DIR os.environ['BUCKET_PREFIX'] = 'byoda' os.environ['CLOUD'] = 'LOCAL' os.environ['NETWORK'] = 'byoda.net' os.environ['ACCOUNT_ID'] = str(get_test_uuid()) os.environ['ACCOUNT_SECRET'] = 'test' os.environ['LOGLEVEL'] = 'DEBUG' os.environ['PRIVATE_KEY_SECRET'] = 'byoda' os.environ['BOOTSTRAP'] = 'BOOTSTRAP' # Remaining environment variables used: network_data = get_environment_vars() network = Network(network_data, network_data) await network.load_network_secrets() config.server = PodServer() config.server.network = network
async def asyncSetUp(self): Logger.getLogger(sys.argv[0], debug=True, json_out=False) with open(CONFIG_FILE) as file_desc: TestKVCache.APP_CONFIG = yaml.load(file_desc, Loader=yaml.SafeLoader) app_config = TestKVCache.APP_CONFIG test_dir = app_config['svcserver']['root_dir'] try: shutil.rmtree(test_dir) except FileNotFoundError: pass os.makedirs(test_dir) # Create the network so that the constructor of ServiceServer # can load it. network: Network = await Network.create( app_config['application']['network'], app_config['svcserver']['root_dir'], app_config['svcserver']['private_key_password']) config.server = ServiceServer(network, app_config) await config.server.load_network_secrets() member_db = config.server.member_db member_db.service_id = app_config['svcserver']['service_id'] member_db.delete_meta(TEST_MEMBER_UUID) member_db.delete_members_list()
async def asyncSetUp(self): Logger.getLogger(sys.argv[0], debug=True, json_out=False) with open(CONFIG_FILE) as file_desc: TestKVCache.APP_CONFIG = yaml.load(file_desc, Loader=yaml.SafeLoader) app_config = TestKVCache.APP_CONFIG test_dir = app_config['svcserver']['root_dir'] try: shutil.rmtree(test_dir) except FileNotFoundError: pass os.makedirs(test_dir) network = await Network.create( app_config['application']['network'], app_config['svcserver']['root_dir'], app_config['svcserver']['private_key_password']) await network.load_network_secrets() network = Network(app_config['svcserver'], app_config['application']) await network.load_network_secrets() config.server = ServiceServer(network, app_config) await config.server.load_network_secrets() config.server.member_db.kvcache.delete(TEST_KEY)
async def setup(): config_file = os.environ.get('CONFIG_FILE', 'config.yml') with open(config_file) as file_desc: app_config = yaml.load(file_desc, Loader=yaml.SafeLoader) debug = app_config['application']['debug'] verbose = not debug _LOGGER = Logger.getLogger(sys.argv[0], debug=debug, verbose=verbose, logfile=app_config['svcserver'].get('logfile')) _LOGGER.debug(f'Read configuration file: {config_file}') network = Network(app_config['svcserver'], app_config['application']) await network.load_network_secrets() if not os.environ.get('SERVER_NAME') and config.server.network.name: os.environ['SERVER_NAME'] = config.server.network.name config.server = ServiceServer(network, app_config) await config.server.load_network_secrets() await config.server.load_secrets( app_config['svcserver']['private_key_password']) await config.server.load_schema() await config.server.service.register_service()
async def setup(): with open('config.yml') as file_desc: app_config = yaml.load(file_desc, Loader=yaml.SafeLoader) debug = app_config['application']['debug'] verbose = not debug global _LOGGER _LOGGER = Logger.getLogger( sys.argv[0], debug=debug, verbose=verbose, logfile=app_config['dirserver'].get('logfile') ) network = Network( app_config['dirserver'], app_config['application'] ) await network.load_network_secrets() server = DirectoryServer(network) await server.connect_db(app_config['dirserver']['dnsdb']) config.server = server await server.get_registered_services() await server.load_secrets() if not os.environ.get('SERVER_NAME') and config.server.network.name: os.environ['SERVER_NAME'] = config.server.network.name
async def asyncSetUp(self): Logger.getLogger(sys.argv[0], debug=True, json_out=False) with open(CONFIG_FILE) as file_desc: TestDirectoryApis.APP_CONFIG = yaml.load(file_desc, Loader=yaml.SafeLoader) app_config = TestDirectoryApis.APP_CONFIG app_config['dirserver']['root_dir'] = TEST_DIR try: shutil.rmtree(TEST_DIR) except FileNotFoundError: pass os.makedirs(TEST_DIR) os.makedirs( f'{SERVICE_DIR}/network-{app_config["application"]["network"]}' f'/services/service-{SERVICE_ID}') network = await Network.create( app_config['application']['network'], app_config['dirserver']['root_dir'], app_config['dirserver']['private_key_password'], ) config.server = DirectoryServer(network) await config.server.connect_db(app_config['dirserver']['dnsdb']) app = setup_api('Byoda test dirserver', 'server for testing directory APIs', 'v0.0.1', [], [account, service, member]) TestDirectoryApis.PROCESS = Process(target=uvicorn.run, args=(app, ), kwargs={ 'host': '127.0.0.1', 'port': TEST_PORT, 'log_level': 'debug' }, daemon=True) TestDirectoryApis.PROCESS.start() await asyncio.sleep(1)
async def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--debug', '-d', action='store_true', default=False) parser.add_argument('--verbose', '-v', action='store_true', default=False) parser.add_argument('--schema', '-s', type=str) parser.add_argument('--network', '-n', type=str, default='test') parser.add_argument('--root-directory', '-r', type=str, default=_ROOT_DIR) parser.add_argument('--password', '-p', type=str, default='byoda') parser.add_argument('--local', default=True, action='store_false') args = parser.parse_args() # Network constructor expects parameters to be in a dict network_data = { 'private_key_password': args.password, 'cloud': 'LOCAL', 'bucket_prefix': None, 'roles': [], 'network': args.network, 'root_dir': args.root_directory, } global _LOGGER _LOGGER = Logger.getLogger(argv[0], debug=args.debug, verbose=args.verbose, json_out=False) root_dir = args.root_directory if root_dir.startswith('/tmp') and os.path.exists(root_dir): _LOGGER.debug(f'Wiping temporary root directory: {root_dir}') shutil.rmtree(root_dir) network_dir = f'{root_dir}/network-{args.network}' network_cert_filepath = (network_dir + f'/network-{args.network}-root-ca-cert.pem') if not os.path.exists(network_cert_filepath): os.makedirs(network_dir, exist_ok=True) resp = requests.get(f'https://dir.{args.network}/root-ca.pem') with open(network_cert_filepath, 'w') as file_desc: file_desc.write(await resp.text()) network = load_network(args, network_data) service = Service(network=network) if args.schema: await service.examine_servicecontract(args.schema) await service.create_secrets(network.services_ca, password=args.password, local=False)
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--debug', '-d', action='store_true', default=False) parser.add_argument('--verbose', '-v', action='store_true', default=False) parser.add_argument('--network', '-n', type=str, default='testdomain.com') parser.add_argument('--root-directory', '-r', type=str, default=_ROOT_DIR) parser.add_argument('--password', '-p', type=str, default='byoda') args = parser.parse_args() global _LOGGER _LOGGER = Logger.getLogger( argv[0], debug=args.debug, verbose=args.verbose, json_out=False ) root_dir = args.root_directory if root_dir.startswith('/tmp') and os.path.exists(root_dir): _LOGGER.debug(f'Wiping temporary root directory: {root_dir}') shutil.rmtree(root_dir) _LOGGER.debug( f'Creating root CA cert and private key under {args.root_directory}' ) Network.create(args.network, root_dir, args.password)
test.assertFalse(exists) subdirs = await storage.get_folders('test/') test.assertEqual(len(subdirs), 2) subdirs = await storage.get_folders('test/', prefix='sub') test.assertEqual(len(subdirs), 1) if (type(storage) in (AzureFileStorage, AwsFileStorage, GcpFileStorage)): url = storage.get_url() + 'test/profile' response = requests.get(url, allow_redirects=False) test.assertIn(response.status_code, (302, 403, 409)) await storage.delete('test/profile') await storage.delete('test/anothersubdir/profile-write') await storage.delete('test/subdir/profile-write') # GCP delete also deletes empty parent 'folders'?? if not type(storage) in (AzureFileStorage, GcpFileStorage): await storage.delete('test/subdir/') await storage.delete('test/anothersubdir/') await storage.delete('test') await storage.close_clients() if __name__ == '__main__': _LOGGER = Logger.getLogger(sys.argv[0], debug=True, json_out=False) unittest.main()
async def main(): config_file = os.environ.get('CONFIG_FILE', 'config.yml') with open(config_file) as file_desc: app_config = yaml.load(file_desc, Loader=yaml.SafeLoader) global _LOGGER debug = app_config['application']['debug'] _LOGGER = Logger.getLogger( sys.argv[0], json_out=True, debug=app_config['application'].get('debug', False), loglevel=app_config['application'].get('loglevel', 'INFO'), logfile=app_config['svcserver'].get('worker_logfile')) if debug: global MAX_WAIT MAX_WAIT = 10 network = Network(app_config['dirserver'], app_config['application']) server = ServiceServer(network, app_config) await server.load_network_secrets() await server.load_secrets( password=app_config['svcserver']['private_key_password']) config.server = server service = server.service try: unprotected_key_file = service.tls_secret.save_tmp_private_key() except PermissionError: _LOGGER.info('Could not write unprotected key, probably because it ' 'already exists') unprotected_key_file = f'/tmp/service-{service.service_id}.key' certkey = (service.paths.root_directory + '/' + service.tls_secret.cert_file, unprotected_key_file) root_ca_certfile = ( f'{service.paths.root_directory}/{service.network.root_ca.cert_file}') if not await service.paths.service_file_exists(service.service_id): await service.download_schema(save=True) await server.load_schema(verify_contract_signatures=False) _LOGGER.debug( f'Starting service worker for service ID: {service.service_id}') while True: member_id = server.member_db.get_next(timeout=MAX_WAIT) if not member_id: _LOGGER.debug('No member available in list of members') continue server.member_db.add_member(member_id) _LOGGER.debug(f'Processing member_id {member_id}') try: data = server.member_db.get_meta(member_id) except (TypeError, KeyError): _LOGGER.warning(f'Invalid data for member: {member_id}') continue server.member_db.add_meta(data['member_id'], data['remote_addr'], data['schema_version'], data['data_secret'], data['status']) waittime = next_member_wait(data['last_seen']) # # Here is where we can do stuff # url = BASE_URL.format(member_id=str(member_id), service_id=service.service_id, network=service.network.name) client = GraphqlClient(endpoint=url, cert=certkey, verify=root_ca_certfile) try: result = client.execute(query=CLIENT_QUERY) if result.get('data'): person_data = result['data']['person'] server.member_db.set_data(member_id, person_data) server.member_db.kvcache.set(person_data['email'], str(member_id)) else: _LOGGER.debug( f'GraphQL person query failed against member {member_id}') except (HTTPError, RequestConnectionError) as exc: _LOGGER.debug(f'Failed to connect to {url}: {exc}') continue # # and now we wait for the time to process the next client # asyncio.sleep(waittime)
async def setup(): server: PodServer = PodServer() config.server = server # Remaining environment variables used: network_data = get_environment_vars() if str(network_data['debug']).lower() == 'true': config.debug = True global _LOGGER _LOGGER = Logger.getLogger(sys.argv[0], json_out=False, debug=config.debug, loglevel=network_data['loglevel'], logfile=LOG_FILE) await server.set_document_store( DocumentStoreType.OBJECT_STORE, cloud_type=CloudType(network_data['cloud']), bucket_prefix=network_data['bucket_prefix'], root_dir=network_data['root_dir']) network = Network(network_data, network_data) await network.load_network_secrets() server.network = network server.paths = network.paths await server.get_registered_services() # TODO: if we have a pod secret, should we compare its commonname with the # account_id environment variable? pod_account = Account(network_data['account_id'], network) await pod_account.paths.create_account_directory() await pod_account.load_memberships() pod_account.password = network_data.get('account_secret') await pod_account.tls_secret.load(password=pod_account.private_key_password ) await pod_account.data_secret.load( password=pod_account.private_key_password) await pod_account.register() server.account = pod_account nginx_config = NginxConfig( directory=NGINX_SITE_CONFIG_DIR, filename='virtualserver.conf', identifier=network_data['account_id'], subdomain=IdType.ACCOUNT.value, cert_filepath=(server.paths.root_directory + '/' + pod_account.tls_secret.cert_file), key_filepath=pod_account.tls_secret.unencrypted_private_key_file, alias=network.paths.account, network=network.name, public_cloud_endpoint=network.paths.storage_driver.get_url( StorageType.PUBLIC), private_cloud_endpoint=network.paths.storage_driver.get_url( StorageType.PRIVATE), port=PodServer.HTTP_PORT, root_dir=server.network.paths.root_directory) nginx_config.create(htaccess_password=pod_account.password) nginx_config.reload() for account_member in pod_account.memberships.values(): account_member.enable_graphql_api(app) await account_member.update_registration()
async def asyncSetUp(self): Logger.getLogger(sys.argv[0], debug=True, json_out=False) with open(CONFIG_FILE) as file_desc: TestDirectoryApis.APP_CONFIG = yaml.load(file_desc, Loader=yaml.SafeLoader) app_config = TestDirectoryApis.APP_CONFIG app_config['svcserver']['service_id'] = SERVICE_ID app_config['svcserver']['root_dir'] = TEST_DIR try: shutil.rmtree(TEST_DIR) except FileNotFoundError: pass os.makedirs(TEST_DIR) service_dir = (f'{TEST_DIR}/network-' f'{app_config["application"]["network"]}' f'/services/service-{SERVICE_ID}') os.makedirs(service_dir) network = await Network.create( app_config['application']['network'], TEST_DIR, app_config['svcserver']['private_key_password']) service_file = network.paths.get(Paths.SERVICE_FILE, service_id=SERVICE_ID) shutil.copy(DUMMY_SCHEMA, TEST_DIR + '/' + service_file) svc = Service(network=network, service_id=app_config['svcserver']['service_id']) if service_file: await svc.examine_servicecontract(service_file) await svc.create_secrets( network.services_ca, local=True, password=app_config['svcserver']['private_key_password']) config.server = ServiceServer(network, app_config) await config.server.load_network_secrets() await config.server.load_secrets( app_config['svcserver']['private_key_password']) await config.server.load_schema(verify_contract_signatures=False) app = setup_api('Byoda test svcserver', 'server for testing service APIs', 'v0.0.1', [], [service, member]) TestDirectoryApis.PROCESS = Process(target=uvicorn.run, args=(app, ), kwargs={ 'host': '127.0.0.1', 'port': TEST_PORT, 'log_level': 'debug' }, daemon=True) TestDirectoryApis.PROCESS.start() await asyncio.sleep(1)
async def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--debug', '-d', action='store_true', default=False) parser.add_argument('--verbose', '-v', action='store_true', default=False) parser.add_argument( '--signing-party', '-s', type=str, default=None, choices=[i.value for i in SignatureType] ) parser.add_argument('--contract', '-c', type=str) parser.add_argument('--config', '-o', type=str, default='config.yml') parser.add_argument('--local', default=False, action='store_true') args = parser.parse_args() # Network constructor expects parameters to be in a dict with open(args.config) as file_desc: app_config = yaml.load(file_desc, Loader=yaml.SafeLoader) root_dir = app_config['svcserver']['root_dir'] password = app_config['svcserver']['private_key_password'] global _LOGGER _LOGGER = Logger.getLogger( argv[0], debug=args.debug, verbose=args.verbose, json_out=False ) network = Network( app_config['svcserver'], app_config['application'] ) config.server = ServiceServer(network, app_config) await config.server.load_network_secrets() service = await load_service(args, config.server.network, password) if not args.local: service.registration_status = await service.get_registration_status() if service.registration_status == RegistrationStatus.Unknown: raise ValueError( 'Please use "create_service_secrets.py" script first' ) schema = service.schema.json_schema if 'signatures' not in schema: schema['signatures'] = {} if not args.local: response = await service.register_service() if response.status != 200: raise ValueError( f'Failed to register service: {response.status}' ) result = None if (not args.signing_party or args.signing_party == SignatureType.SERVICE.value): try: create_service_signature(service) result = True except Exception as exc: _LOGGER.exception(f'Failed to get service signature: {exc}') result = False if (result is not False and ( not args.signing_party or args.signing_party == SignatureType.NETWORK.value)): result = await create_network_signature(service, args, password) if not result: _LOGGER.error('Failed to get the network signature') sys.exit(1) if not root_dir.startswith('/'): # Hack to make relative paths work with the FileStorage class root_dir = os.getcwd() storage_driver = FileStorage(root_dir) filepath = service.paths.get(Paths.SERVICE_FILE) _LOGGER.debug(f'Saving signed schema to {filepath}') await service.schema.save(filepath, storage_driver=storage_driver)