def db_client(): """ Maintain instance of DB Client """ config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath='tests/artifacts/db_client/configuration.ini') return DBClient(config_manager)
def bootstrap(): """ Bootstraps the data provider unix service. It instantiates the Configuration Manager, Dataset Manager, Communication Manager and the Execution Pipeline. """ # 1. Set up Configuration Manager. config_manager = ConfigurationManager() config_manager.bootstrap() # 2. Set up the IPFS Client used by the service config = config_manager.get_config() client = None try: client = ipfsapi.connect(config.get('BLOCKCHAIN', 'host'), config.getint('BLOCKCHAIN', 'ipfs_port')) except Exception as e: # TODO: Can this log the exception? # logging.info("IPFS daemon not started, got: {0}".format(e)) raise (e) # 2. Set up Dataset Manager. dataset_manager = DatasetManager(config_manager=config_manager) dataset_manager.configure(ipfs_client=client) # 3. Set up the Communication Manager. communication_manager = CommunicationManager() # 4. Set up the Execution Pipeline (Scheduler, Runners) # and run the Scheduler's cron on a new thread. scheduler = DMLScheduler(config_manager=config_manager, ) scheduler.configure(communication_manager=communication_manager, ipfs_client=client) t1 = threading.Thread( target=scheduler.start_cron, args=(0.05, ), daemon=False, ) t1.start() # 5. Configure the Communication Manager with the components it talks to. communication_manager.configure(scheduler=scheduler) # 6. Set up Blockchain Gateway and start listening on a new thread. blockchain_gateway = BlockchainGateway() blockchain_gateway.configure(config_manager=config_manager, communication_manager=communication_manager, ipfs_client=client) t2 = threading.Thread( target=blockchain_gateway.start_cron, args=(0.05, ), daemon=False, ) t2.start() # 7. Wait for the threads to end. # TODO: Need to make it work as a daemon. t1.join()
def test_multiple_secret_sections(multiple_secret_config_filepath, \ multiple_secret_questions_filepath, multiple_secret_arr): config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath=multiple_secret_config_filepath, input_function=lambda x: '', question_filepath=multiple_secret_questions_filepath, ) check_config_exists(config_manager) setup_default_worked(config_manager) verify_secret_section_values(config_manager, multiple_secret_arr) comments_test(multiple_secret_config_filepath, multiple_secret_arr) os.remove(config_manager.config_filepath)
def test_complete_setup_default(default_config_filepath, questions_filepath, \ single_secret_arr): """ Verify configuration.ini from default user input is created and correct. """ config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath=default_config_filepath, input_function=lambda x: '', question_filepath=questions_filepath ) check_config_exists(config_manager) setup_default_worked(config_manager) verify_secret_section_values(config_manager, single_secret_arr) comments_test(default_config_filepath, single_secret_arr) os.remove(config_manager.config_filepath)
def test_no_setup_repeat(no_repeat_config_filepath, questions_filepath, \ single_secret_arr): """ Verify that run_setup_mode is not run again when configuration already exists. Or more simply, that the configuration has not changed) """ config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath=no_repeat_config_filepath, input_function=lambda x: '', question_filepath=questions_filepath ) check_config_exists(config_manager) assert not config_manager.bootstrap() setup_default_worked(config_manager) verify_secret_section_values(config_manager, single_secret_arr) os.remove(config_manager.config_filepath)
def test_complete_setup_custom(custom_config_filepath, questions_filepath, \ single_secret_arr): """ Verify configuration.ini from custom user input is created and correct. """ config_manager = ConfigurationManager() custom_string = 'test' config_manager.bootstrap( config_filepath=custom_config_filepath, input_function=lambda x: custom_string, question_filepath=questions_filepath ) check_config_exists(config_manager) setup_custom_worked(config_manager, custom_string) verify_secret_section_values(config_manager, single_secret_arr) comments_test(custom_config_filepath, single_secret_arr) os.remove(config_manager.config_filepath)
def bootstrap(repo_id="testRepo", api_key="demo-api-key", test=False): """ Bootstraps the data provider unix service. It instantiates the Configuration Manager, Dataset Manager, Communication Manager and the Execution Pipeline. """ # 1. Set up Configuration Manager. config_manager = ConfigurationManager() config_manager.bootstrap() runner = DMLRunner(config_manager) optimizer = FederatedAveragingOptimizer(runner, repo_id) loop = asyncio.get_event_loop() websocket_client = WebSocketClient(optimizer, config_manager, repo_id, api_key, test) # mappings = dataset_manager.get_mappings() # 7. Wait for the threads to end. # TODO: Need to make it work as a daemon. loop.run_until_complete(websocket_client.prepare_dml())
def bad_config_manager_header(): config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath='tests/artifacts/dataset_manager/configuration3.ini') return config_manager
def config_manager(): config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath='tests/artifacts/runner_scheduler/configuration.ini' ) return config_manager
def config(): config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath='tests/artifacts/blockchain/configuration.ini') return config_manager.get_config()
def config_manager(): config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath='tests/artifacts/fed_avg_optimizer/configuration.ini') return config_manager
def config_manager_two(): config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath='tests/artifacts/integration/configuration2.ini') return config_manager
def config_manager(): config_manager = ConfigurationManager() config_manager.bootstrap( config_filepath= 'tests/artifacts/communication_manager/configuration.ini') return config_manager