def bootstrap():
"""
Bootstraps the data provider unix service.
It instantiates the Configuration Manager, Dataset Manager, Communication
Manager and the Execution Pipeline.
"""
# 1. Set up Configuration Manager.
config_manager = ConfigurationManager()
config_manager.bootstrap()
# 2. Set up the IPFS Client used by the service
config = config_manager.get_config()
client = None
try:
client = ipfsapi.connect(config.get('BLOCKCHAIN', 'host'),
config.getint('BLOCKCHAIN', 'ipfs_port'))
except Exception as e:
# TODO: Can this log the exception?
# logging.info("IPFS daemon not started, got: {0}".format(e))
raise (e)
# 2. Set up Dataset Manager.
dataset_manager = DatasetManager(config_manager=config_manager)
dataset_manager.configure(ipfs_client=client)
# 3. Set up the Communication Manager.
communication_manager = CommunicationManager()
# 4. Set up the Execution Pipeline (Scheduler, Runners)
# and run the Scheduler's cron on a new thread.
scheduler = DMLScheduler(config_manager=config_manager, )
scheduler.configure(communication_manager=communication_manager,
ipfs_client=client)
t1 = threading.Thread(
target=scheduler.start_cron,
args=(0.05, ),
daemon=False,
)
t1.start()
# 5. Configure the Communication Manager with the components it talks to.
communication_manager.configure(scheduler=scheduler)
# 6. Set up Blockchain Gateway and start listening on a new thread.
blockchain_gateway = BlockchainGateway()
blockchain_gateway.configure(config_manager=config_manager,
communication_manager=communication_manager,
ipfs_client=client)
t2 = threading.Thread(
target=blockchain_gateway.start_cron,
args=(0.05, ),
daemon=False,
)
t2.start()
# 7. Wait for the threads to end.
# TODO: Need to make it work as a daemon.
t1.join()
def setup_client(config_manager, client):
"""
Set up and return communication_manager, blockchain_gateway, scheduler
"""
communication_manager = CommunicationManager()
blockchain_gateway = BlockchainGateway()
scheduler = DMLScheduler(config_manager)
communication_manager.configure(scheduler)
blockchain_gateway.configure(config_manager, communication_manager, client)
scheduler.configure(communication_manager, client)
return communication_manager, blockchain_gateway, scheduler
def setup_client(config_manager, ipfs_client, dataset_manager):
class MockGateway(object):
def __init__(self):
self.state = []
communication_manager = CommunicationManager()
scheduler = DMLScheduler(config_manager)
blockchain_gateway = MockGateway()
communication_manager.configure(scheduler, dataset_manager)
scheduler.configure(communication_manager, ipfs_client, blockchain_gateway)
return communication_manager, scheduler
def test_communication_manager_fails_if_not_configured(new_session_event):
"""
Ensures that Communication Manager won't be able to function if it's not
configured.
"""
communication_manager = CommunicationManager()
try:
communication_manager.inform(MessageEventTypes.NEW_SESSION.name,
new_session_event)
raise Exception("This should have raised an exception")
except Exception as e:
assert str(e) == "Communication Manager needs to be configured first!"
def setup_client(config_manager, client):
"""
Set up and return communication_manager, blockchain_gateway, scheduler
"""
dataset_manager = DatasetManager(config_manager)
dataset_manager.bootstrap()
communication_manager = CommunicationManager()
blockchain_gateway = BlockchainGateway()
blockchain_gateway.configure(config_manager=config_manager,
communication_manager=communication_manager,
ipfs_client=client,
dataset_manager=dataset_manager)
scheduler = DMLScheduler(config_manager)
scheduler.configure(communication_manager=communication_manager,
ipfs_client=client,
blockchain_gateway=blockchain_gateway)
communication_manager.configure(scheduler=scheduler,
dataset_manager=dataset_manager)
return communication_manager, blockchain_gateway, scheduler
def test_communication_manager_can_inform_new_job_to_the_optimizer(
config_manager, ipfs_client):
"""
Ensures that Communication Manager can tell the optimizer of something,
and that the job will transfer correctly.
"""
communication_manager = CommunicationManager()
scheduler = DMLScheduler(config_manager)
communication_manager.configure(scheduler)
scheduler.configure(communication_manager, ipfs_client)
true_job = make_initialize_job(make_model_json())
true_job.hyperparams['epochs'] = 10
true_job.hyperparams['batch_size'] = 128
true_job.hyperparams['split'] = .05
serialized_job = serialize_job(true_job)
new_session_event = {
TxEnum.KEY.name: None,
TxEnum.CONTENT.name: {
"optimizer_params": {},
"serialized_job": serialized_job
}
}
communication_manager.inform(MessageEventTypes.NEW_SESSION.name,
new_session_event)
optimizer_job = communication_manager.optimizer.job
assert optimizer_job.weights == true_job.weights
assert optimizer_job.serialized_model == true_job.serialized_model
assert optimizer_job.framework_type == true_job.framework_type
assert optimizer_job.hyperparams == true_job.hyperparams
assert optimizer_job.label_column_name == true_job.label_column_name
def test_communication_manager_creates_new_sessions(new_session_event,
config_manager,
ipfs_client):
"""
Ensures that upon receiving an initialization job, the Communication Manager
will make an optimizer.
"""
communication_manager = CommunicationManager()
scheduler = DMLScheduler(config_manager)
communication_manager.configure(scheduler)
scheduler.configure(communication_manager, ipfs_client)
communication_manager.inform(MessageEventTypes.NEW_SESSION.name,
new_session_event)
assert communication_manager.optimizer
def test_communication_manager_can_be_initialized():
"""
Very simple check. Checks if the Communication Manager can initialize.
"""
communication_manager = CommunicationManager()
assert communication_manager