Пример #1
0
def test_communication_manager_can_inform_new_job_to_the_optimizer(
        config_manager, ipfs_client):
    """
    Ensures that Communication Manager can tell the optimizer of something,
    and that the job will transfer correctly.
    """
    communication_manager = CommunicationManager()
    scheduler = DMLScheduler(config_manager)
    communication_manager.configure(scheduler)
    scheduler.configure(communication_manager, ipfs_client)
    true_job = make_initialize_job(make_model_json())
    true_job.hyperparams['epochs'] = 10
    true_job.hyperparams['batch_size'] = 128
    true_job.hyperparams['split'] = .05
    serialized_job = serialize_job(true_job)
    new_session_event = {
        TxEnum.KEY.name: None,
        TxEnum.CONTENT.name: {
            "optimizer_params": {},
            "serialized_job": serialized_job
        }
    }
    communication_manager.inform(MessageEventTypes.NEW_SESSION.name,
                                 new_session_event)
    optimizer_job = communication_manager.optimizer.job
    assert optimizer_job.weights == true_job.weights
    assert optimizer_job.serialized_model == true_job.serialized_model
    assert optimizer_job.framework_type == true_job.framework_type
    assert optimizer_job.hyperparams == true_job.hyperparams
    assert optimizer_job.label_column_name == true_job.label_column_name
Пример #2
0
def bootstrap():
    """
    Bootstraps the data provider unix service.

    It instantiates the Configuration Manager, Dataset Manager, Communication
    Manager and the Execution Pipeline.
    """
    # 1. Set up Configuration Manager.
    config_manager = ConfigurationManager()
    config_manager.bootstrap()

    # 2. Set up the IPFS Client used by the service
    config = config_manager.get_config()
    client = None
    try:
        client = ipfsapi.connect(config.get('BLOCKCHAIN', 'host'),
                                 config.getint('BLOCKCHAIN', 'ipfs_port'))
    except Exception as e:
        # TODO: Can this log the exception?
        # logging.info("IPFS daemon not started, got: {0}".format(e))
        raise (e)
    # 2. Set up Dataset Manager.
    dataset_manager = DatasetManager(config_manager=config_manager)
    dataset_manager.configure(ipfs_client=client)

    # 3. Set up the Communication Manager.
    communication_manager = CommunicationManager()

    # 4. Set up the Execution Pipeline (Scheduler, Runners)
    # and run the Scheduler's cron on a new thread.
    scheduler = DMLScheduler(config_manager=config_manager, )
    scheduler.configure(communication_manager=communication_manager,
                        ipfs_client=client)
    t1 = threading.Thread(
        target=scheduler.start_cron,
        args=(0.05, ),
        daemon=False,
    )
    t1.start()

    # 5. Configure the Communication Manager with the components it talks to.
    communication_manager.configure(scheduler=scheduler)

    # 6. Set up Blockchain Gateway and start listening on a new thread.
    blockchain_gateway = BlockchainGateway()
    blockchain_gateway.configure(config_manager=config_manager,
                                 communication_manager=communication_manager,
                                 ipfs_client=client)
    t2 = threading.Thread(
        target=blockchain_gateway.start_cron,
        args=(0.05, ),
        daemon=False,
    )
    t2.start()

    # 7. Wait for the threads to end.
    # TODO: Need to make it work as a daemon.
    t1.join()
Пример #3
0
def setup_client(config_manager, client):
    """
    Set up and return communication_manager, blockchain_gateway, scheduler
    """
    communication_manager = CommunicationManager()
    blockchain_gateway = BlockchainGateway()
    scheduler = DMLScheduler(config_manager)
    communication_manager.configure(scheduler)
    blockchain_gateway.configure(config_manager, communication_manager, client)
    scheduler.configure(communication_manager, client)
    return communication_manager, blockchain_gateway, scheduler
Пример #4
0
def setup_client(config_manager, ipfs_client, dataset_manager):
    class MockGateway(object):
        def __init__(self):
            self.state = []

    communication_manager = CommunicationManager()
    scheduler = DMLScheduler(config_manager)
    blockchain_gateway = MockGateway()
    communication_manager.configure(scheduler, dataset_manager)
    scheduler.configure(communication_manager, ipfs_client, blockchain_gateway)
    return communication_manager, scheduler
Пример #5
0
def test_communication_manager_creates_new_sessions(new_session_event,
                                                    config_manager,
                                                    ipfs_client):
    """
    Ensures that upon receiving an initialization job, the Communication Manager
    will make an optimizer.
    """
    communication_manager = CommunicationManager()
    scheduler = DMLScheduler(config_manager)
    communication_manager.configure(scheduler)
    scheduler.configure(communication_manager, ipfs_client)
    communication_manager.inform(MessageEventTypes.NEW_SESSION.name,
                                 new_session_event)
    assert communication_manager.optimizer
def setup_client(config_manager, client):
    """
    Set up and return communication_manager, blockchain_gateway, scheduler
    """
    dataset_manager = DatasetManager(config_manager)
    dataset_manager.bootstrap()
    communication_manager = CommunicationManager()
    blockchain_gateway = BlockchainGateway()
    blockchain_gateway.configure(config_manager=config_manager,
                                 communication_manager=communication_manager,
                                 ipfs_client=client,
                                 dataset_manager=dataset_manager)
    scheduler = DMLScheduler(config_manager)
    scheduler.configure(communication_manager=communication_manager,
                        ipfs_client=client,
                        blockchain_gateway=blockchain_gateway)
    communication_manager.configure(scheduler=scheduler,
                                    dataset_manager=dataset_manager)
    return communication_manager, blockchain_gateway, scheduler
Пример #7
0
def scheduler(config_manager, ipfs_client, communication_manager, dataset_manager, blockchain_gateway):
    scheduler = DMLScheduler(config_manager)
    scheduler.configure(communication_manager, ipfs_client, blockchain_gateway)
    communication_manager.configure(scheduler, dataset_manager)
    return scheduler
def scheduler(config_manager, ipfs_client, communication_manager):
    scheduler = DMLScheduler(config_manager)
    scheduler.configure(communication_manager, ipfs_client)
    return scheduler