def db_client():
    """
    Maintain instance of DB Client
    """
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath='tests/artifacts/db_client/configuration.ini')
    return DBClient(config_manager)
Exemple #2
0
def bootstrap():
    """
    Bootstraps the data provider unix service.

    It instantiates the Configuration Manager, Dataset Manager, Communication
    Manager and the Execution Pipeline.
    """
    # 1. Set up Configuration Manager.
    config_manager = ConfigurationManager()
    config_manager.bootstrap()

    # 2. Set up the IPFS Client used by the service
    config = config_manager.get_config()
    client = None
    try:
        client = ipfsapi.connect(config.get('BLOCKCHAIN', 'host'),
                                 config.getint('BLOCKCHAIN', 'ipfs_port'))
    except Exception as e:
        # TODO: Can this log the exception?
        # logging.info("IPFS daemon not started, got: {0}".format(e))
        raise (e)
    # 2. Set up Dataset Manager.
    dataset_manager = DatasetManager(config_manager=config_manager)
    dataset_manager.configure(ipfs_client=client)

    # 3. Set up the Communication Manager.
    communication_manager = CommunicationManager()

    # 4. Set up the Execution Pipeline (Scheduler, Runners)
    # and run the Scheduler's cron on a new thread.
    scheduler = DMLScheduler(config_manager=config_manager, )
    scheduler.configure(communication_manager=communication_manager,
                        ipfs_client=client)
    t1 = threading.Thread(
        target=scheduler.start_cron,
        args=(0.05, ),
        daemon=False,
    )
    t1.start()

    # 5. Configure the Communication Manager with the components it talks to.
    communication_manager.configure(scheduler=scheduler)

    # 6. Set up Blockchain Gateway and start listening on a new thread.
    blockchain_gateway = BlockchainGateway()
    blockchain_gateway.configure(config_manager=config_manager,
                                 communication_manager=communication_manager,
                                 ipfs_client=client)
    t2 = threading.Thread(
        target=blockchain_gateway.start_cron,
        args=(0.05, ),
        daemon=False,
    )
    t2.start()

    # 7. Wait for the threads to end.
    # TODO: Need to make it work as a daemon.
    t1.join()
Exemple #3
0
def test_multiple_secret_sections(multiple_secret_config_filepath, \
    multiple_secret_questions_filepath, multiple_secret_arr):
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath=multiple_secret_config_filepath,
        input_function=lambda x: '',
        question_filepath=multiple_secret_questions_filepath,
    )
    check_config_exists(config_manager)
    setup_default_worked(config_manager)
    verify_secret_section_values(config_manager, multiple_secret_arr)
    comments_test(multiple_secret_config_filepath, multiple_secret_arr)
    os.remove(config_manager.config_filepath)
Exemple #4
0
def test_complete_setup_default(default_config_filepath, questions_filepath, \
    single_secret_arr):
    """
    Verify configuration.ini from default user input is created and correct.
    """
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath=default_config_filepath,
        input_function=lambda x: '',
        question_filepath=questions_filepath
    )
    check_config_exists(config_manager)
    setup_default_worked(config_manager)
    verify_secret_section_values(config_manager, single_secret_arr)
    comments_test(default_config_filepath, single_secret_arr)
    os.remove(config_manager.config_filepath)
Exemple #5
0
def test_no_setup_repeat(no_repeat_config_filepath, questions_filepath, \
    single_secret_arr):
    """
    Verify that run_setup_mode is not run again when configuration already
    exists. Or more simply, that the configuration has not changed) 
    """
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath=no_repeat_config_filepath,
        input_function=lambda x: '',
        question_filepath=questions_filepath
    )
    check_config_exists(config_manager)
    assert not config_manager.bootstrap()
    setup_default_worked(config_manager)
    verify_secret_section_values(config_manager, single_secret_arr)
    os.remove(config_manager.config_filepath)
Exemple #6
0
def test_complete_setup_custom(custom_config_filepath, questions_filepath, \
    single_secret_arr):
    """
    Verify configuration.ini from custom user input is created and correct.
    """
    config_manager = ConfigurationManager()
    custom_string = 'test'
    config_manager.bootstrap(
        config_filepath=custom_config_filepath,
        input_function=lambda x: custom_string,
        question_filepath=questions_filepath
    )
    check_config_exists(config_manager)
    setup_custom_worked(config_manager, custom_string)
    verify_secret_section_values(config_manager, single_secret_arr)
    comments_test(custom_config_filepath, single_secret_arr)
    os.remove(config_manager.config_filepath)
Exemple #7
0
def bootstrap(repo_id="testRepo", api_key="demo-api-key", test=False):
    """
    Bootstraps the data provider unix service.

    It instantiates the Configuration Manager, Dataset Manager, Communication
    Manager and the Execution Pipeline.
    """
    # 1. Set up Configuration Manager.
    config_manager = ConfigurationManager()
    config_manager.bootstrap()

    runner = DMLRunner(config_manager)

    optimizer = FederatedAveragingOptimizer(runner, repo_id)

    loop = asyncio.get_event_loop()

    websocket_client = WebSocketClient(optimizer, config_manager, repo_id,
                                       api_key, test)
    # mappings = dataset_manager.get_mappings()

    # 7. Wait for the threads to end.
    # TODO: Need to make it work as a daemon.
    loop.run_until_complete(websocket_client.prepare_dml())
Exemple #8
0
def bad_config_manager_header():
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath='tests/artifacts/dataset_manager/configuration3.ini')
    return config_manager
Exemple #9
0
def config_manager():
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath='tests/artifacts/runner_scheduler/configuration.ini'
    )
    return config_manager
Exemple #10
0
def config():
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath='tests/artifacts/blockchain/configuration.ini')
    return config_manager.get_config()
Exemple #11
0
def config_manager():
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath='tests/artifacts/fed_avg_optimizer/configuration.ini')
    return config_manager
def config_manager_two():
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath='tests/artifacts/integration/configuration2.ini')
    return config_manager
Exemple #13
0
def config_manager():
    config_manager = ConfigurationManager()
    config_manager.bootstrap(
        config_filepath=
        'tests/artifacts/communication_manager/configuration.ini')
    return config_manager