def db_client():
"""
Maintain instance of DB Client
"""
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath='tests/artifacts/db_client/configuration.ini')
return DBClient(config_manager)
def bootstrap():
"""
Bootstraps the data provider unix service.
It instantiates the Configuration Manager, Dataset Manager, Communication
Manager and the Execution Pipeline.
"""
# 1. Set up Configuration Manager.
config_manager = ConfigurationManager()
config_manager.bootstrap()
# 2. Set up the IPFS Client used by the service
config = config_manager.get_config()
client = None
try:
client = ipfsapi.connect(config.get('BLOCKCHAIN', 'host'),
config.getint('BLOCKCHAIN', 'ipfs_port'))
except Exception as e:
# TODO: Can this log the exception?
# logging.info("IPFS daemon not started, got: {0}".format(e))
raise (e)
# 2. Set up Dataset Manager.
dataset_manager = DatasetManager(config_manager=config_manager)
dataset_manager.configure(ipfs_client=client)
# 3. Set up the Communication Manager.
communication_manager = CommunicationManager()
# 4. Set up the Execution Pipeline (Scheduler, Runners)
# and run the Scheduler's cron on a new thread.
scheduler = DMLScheduler(config_manager=config_manager, )
scheduler.configure(communication_manager=communication_manager,
ipfs_client=client)
t1 = threading.Thread(
target=scheduler.start_cron,
args=(0.05, ),
daemon=False,
)
t1.start()
# 5. Configure the Communication Manager with the components it talks to.
communication_manager.configure(scheduler=scheduler)
# 6. Set up Blockchain Gateway and start listening on a new thread.
blockchain_gateway = BlockchainGateway()
blockchain_gateway.configure(config_manager=config_manager,
communication_manager=communication_manager,
ipfs_client=client)
t2 = threading.Thread(
target=blockchain_gateway.start_cron,
args=(0.05, ),
daemon=False,
)
t2.start()
# 7. Wait for the threads to end.
# TODO: Need to make it work as a daemon.
t1.join()
def test_multiple_secret_sections(multiple_secret_config_filepath, \
multiple_secret_questions_filepath, multiple_secret_arr):
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath=multiple_secret_config_filepath,
input_function=lambda x: '',
question_filepath=multiple_secret_questions_filepath,
)
check_config_exists(config_manager)
setup_default_worked(config_manager)
verify_secret_section_values(config_manager, multiple_secret_arr)
comments_test(multiple_secret_config_filepath, multiple_secret_arr)
os.remove(config_manager.config_filepath)
def test_complete_setup_default(default_config_filepath, questions_filepath, \
single_secret_arr):
"""
Verify configuration.ini from default user input is created and correct.
"""
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath=default_config_filepath,
input_function=lambda x: '',
question_filepath=questions_filepath
)
check_config_exists(config_manager)
setup_default_worked(config_manager)
verify_secret_section_values(config_manager, single_secret_arr)
comments_test(default_config_filepath, single_secret_arr)
os.remove(config_manager.config_filepath)
def test_no_setup_repeat(no_repeat_config_filepath, questions_filepath, \
single_secret_arr):
"""
Verify that run_setup_mode is not run again when configuration already
exists. Or more simply, that the configuration has not changed)
"""
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath=no_repeat_config_filepath,
input_function=lambda x: '',
question_filepath=questions_filepath
)
check_config_exists(config_manager)
assert not config_manager.bootstrap()
setup_default_worked(config_manager)
verify_secret_section_values(config_manager, single_secret_arr)
os.remove(config_manager.config_filepath)
def test_complete_setup_custom(custom_config_filepath, questions_filepath, \
single_secret_arr):
"""
Verify configuration.ini from custom user input is created and correct.
"""
config_manager = ConfigurationManager()
custom_string = 'test'
config_manager.bootstrap(
config_filepath=custom_config_filepath,
input_function=lambda x: custom_string,
question_filepath=questions_filepath
)
check_config_exists(config_manager)
setup_custom_worked(config_manager, custom_string)
verify_secret_section_values(config_manager, single_secret_arr)
comments_test(custom_config_filepath, single_secret_arr)
os.remove(config_manager.config_filepath)
def bootstrap(repo_id="testRepo", api_key="demo-api-key", test=False):
"""
Bootstraps the data provider unix service.
It instantiates the Configuration Manager, Dataset Manager, Communication
Manager and the Execution Pipeline.
"""
# 1. Set up Configuration Manager.
config_manager = ConfigurationManager()
config_manager.bootstrap()
runner = DMLRunner(config_manager)
optimizer = FederatedAveragingOptimizer(runner, repo_id)
loop = asyncio.get_event_loop()
websocket_client = WebSocketClient(optimizer, config_manager, repo_id,
api_key, test)
# mappings = dataset_manager.get_mappings()
# 7. Wait for the threads to end.
# TODO: Need to make it work as a daemon.
loop.run_until_complete(websocket_client.prepare_dml())
def bad_config_manager_header():
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath='tests/artifacts/dataset_manager/configuration3.ini')
return config_manager
def config_manager():
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath='tests/artifacts/runner_scheduler/configuration.ini'
)
return config_manager
def config():
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath='tests/artifacts/blockchain/configuration.ini')
return config_manager.get_config()
def config_manager():
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath='tests/artifacts/fed_avg_optimizer/configuration.ini')
return config_manager
def config_manager_two():
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath='tests/artifacts/integration/configuration2.ini')
return config_manager
def config_manager():
config_manager = ConfigurationManager()
config_manager.bootstrap(
config_filepath=
'tests/artifacts/communication_manager/configuration.ini')
return config_manager