Esempio n. 1
0
def generate_go_ethereum_fixture(destination_dir):
    with contextlib.ExitStack() as stack:
        datadir = stack.enter_context(common.tempdir())

        keystore_dir = os.path.join(datadir, 'keystore')
        common.ensure_path_exists(keystore_dir)
        keyfile_path = os.path.join(keystore_dir, common.KEYFILE_FILENAME)
        with open(keyfile_path, 'w') as keyfile:
            keyfile.write(common.KEYFILE_DATA)
        genesis_file_path = os.path.join(datadir, 'genesis.json')
        with open(genesis_file_path, 'w') as genesis_file:
            genesis_file.write(json.dumps(common.GENESIS_DATA))

        geth_ipc_path_dir = stack.enter_context(common.tempdir())
        geth_ipc_path = os.path.join(geth_ipc_path_dir, 'geth.ipc')

        geth_port = get_open_port()
        geth_binary = common.get_geth_binary()

        with get_geth_process(
                geth_binary=geth_binary,
                datadir=datadir,
                genesis_file_path=genesis_file_path,
                geth_ipc_path=geth_ipc_path,
                geth_port=geth_port):

            common.wait_for_socket(geth_ipc_path)
            w3 = Web3(Web3.IPCProvider(geth_ipc_path))
            chain_data = setup_chain_state(w3)
            # close geth by exiting context
            # must be closed before copying data dir
            verify_chain_state(w3, chain_data)

        # verify that chain state is still valid after closing
        # and re-opening geth
        with get_geth_process(
                geth_binary=geth_binary,
                datadir=datadir,
                genesis_file_path=genesis_file_path,
                geth_ipc_path=geth_ipc_path,
                geth_port=geth_port):

            common.wait_for_socket(geth_ipc_path)
            w3 = Web3(Web3.IPCProvider(geth_ipc_path))
            verify_chain_state(w3, chain_data)

        static_data = {
            'raw_txn_account': common.RAW_TXN_ACCOUNT,
            'keyfile_pw': common.KEYFILE_PW,
        }
        config = merge(chain_data, static_data)
        pprint.pprint(config)
        write_config_json(config, datadir)

        shutil.make_archive(destination_dir, 'zip', datadir)
Esempio n. 2
0
    def test_happy_run_elastic_fault_tolerant(self):
        if skip_lightning_tests:
            self.skipTest(
                'Spark PyTorch Lightning tests conflict with Tensorflow 2.5.x: '
                'https://github.com/horovod/horovod/pull/3263')

        if not gloo_built():
            self.skipTest("Gloo is not available")

        with spark_session('test_happy_run_elastic_fault_tolerant',
                           max_failures=3):
            with tempdir() as dir:
                # these files make training function fail in given rank, epoch and batch
                with open(os.path.sep.join([dir, 'rank_1_epoch_2_batch_4_fail']), 'w'), \
                     open(os.path.sep.join([dir, 'rank_0_epoch_3_batch_1_fail']), 'w'), \
                     open(os.path.sep.join([dir, 'rank_1_epoch_4_batch_2_fail']), 'w'):
                    pass
                res = horovod.spark.run_elastic(
                    fn,
                    args=(2, 5, 5, dir),
                    env={'HOROVOD_LOG_LEVEL': 'DEBUG'},
                    num_proc=2,
                    min_num_proc=2,
                    max_num_proc=2,
                    start_timeout=5,
                    verbose=2)
                self.assertListEqual([([0, 4, 0, 4, 1, 4, 0, 4], 0),
                                      ([0, 4, 0, 4, 1, 4, 0, 4], 1)], res)
Esempio n. 3
0
    def test_model_checkpoint_callback(self):
        from pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint

        with spark_session('test_fit_model') as spark:
            df = create_noisy_xor_data(spark)
            model = create_xor_model()

            with tempdir() as dir:
                checkpoint_callback = ModelCheckpoint(dirpath=dir)
                callbacks = [checkpoint_callback]

                with local_store() as store:
                    torch_estimator = hvd_spark.TorchEstimator(
                        num_proc=2,
                        store=store,
                        model=model,
                        input_shapes=[[-1, 2]],
                        feature_cols=['features'],
                        label_cols=['y'],
                        validation=0.2,
                        batch_size=4,
                        epochs=2,
                        verbose=2,
                        callbacks=callbacks)

                    torch_model = torch_estimator.fit(df)

                    # TODO: Find a way to pass log metrics from remote, and assert base on the logger.
                    trained_model = torch_model.getModel()
                    pred = trained_model(torch.ones([1, 2], dtype=torch.int32))
                    assert len(pred) == 1
                    assert pred.dtype == torch.float32
Esempio n. 4
0
    def test_model_override_trainer_args(self):
        if skip_lightning_tests:
            self.skipTest(
                'Spark PyTorch Lightning tests conflict with Tensorflow 2.5.x: '
                'https://github.com/horovod/horovod/pull/3263')

        from pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint

        with spark_session('test_fit_model') as spark:
            df = create_noisy_xor_data(spark)
            model = create_xor_model()

            with tempdir() as dir:

                with local_store() as store:
                    torch_estimator = hvd_spark.TorchEstimator(
                        num_proc=2,
                        store=store,
                        model=model,
                        input_shapes=[[-1, 2]],
                        feature_cols=['features'],
                        label_cols=['y'],
                        validation=0.2,
                        batch_size=4,
                        epochs=2,
                        verbose=2,
                        trainer_args={'stochastic_weight_avg': True})

                    torch_model = torch_estimator.fit(df)

                    # TODO: Find a way to pass log metrics from remote, and assert base on the logger.
                    trained_model = torch_model.getModel()
                    pred = trained_model(torch.ones([1, 2], dtype=torch.int32))
                    assert len(pred) == 1
                    assert pred.dtype == torch.float32
Esempio n. 5
0
    def test_spark_task_service_env(self):
        key = secret.make_secret_key()
        service_env = dict([(key, '{} value'.format(key))
                            for key in SparkTaskService.SERVICE_ENV_KEYS])
        service_env.update({"other": "value"})
        with os_environ(service_env):
            service = SparkTaskService(1, key, None)
            client = SparkTaskClient(1, service.addresses(), key, 3)

            with tempdir() as d:
                file = '{}/env'.format(d)
                command = "env | grep -v '^PWD='> {}".format(file)
                command_env = {"test": "value"}

                try:
                    client.run_command(command, command_env)
                    client.wait_for_command_termination()
                finally:
                    service.shutdown()

                with open(file) as f:
                    env = sorted([line.strip() for line in f.readlines()])
                    expected = [
                        'HADOOP_TOKEN_FILE_LOCATION=HADOOP_TOKEN_FILE_LOCATION value',
                        'test=value'
                    ]
                    self.assertEqual(env, expected)
Esempio n. 6
0
    def test_gen_pipeline_with_non_code_changes(self):
        with tempdir() as dir:
            tmp_gen_pipeline_sh = os.path.join(dir, 'gen-pipeline.sh')
            copy(GEN_PIPELINE_FNAME, tmp_gen_pipeline_sh)

            with open(os.path.join(dir, 'get_changed_code_files.py'),
                      'w') as py:
                py.write("pass")

            gen_pipeline_env = dict(BUILDKITE_PIPELINE_SLUG='SLUG',
                                    BUILDKITE_BRANCH='BRANCH')
            exit_code, actual_pipeline, gen_pipeline_log = self._run(
                tmp_gen_pipeline_sh, gen_pipeline_env)

            self.assertEqual(0, exit_code)
            self.assertEqual('', gen_pipeline_log)
            self.assertEqual(
                "steps:\n"
                "- label: \':book: Build Docs\'\n"
                "  command: 'cd /workdir/docs && pip install -r requirements.txt && make html'\n"
                "  plugins:\n"
                "  - docker#v3.1.0:\n"
                "      image: 'python:3.7'\n"
                "  timeout_in_minutes: 5\n"
                "  retry:\n"
                "    automatic: true\n"
                "  agents:\n"
                "    queue: cpu\n"
                "- wait\n"
                "- wait\n"
                "- wait\n", actual_pipeline)
Esempio n. 7
0
    def test_happy_run_elastic_fault_tolerant_fails(self):
        self.skipTest(
            'elastic horovod does not support shutdown from the spark driver '
            'while elastic driver is waiting for hosts to come up')

        if not gloo_built():
            self.skipTest("Gloo is not available")

        with spark_session('test_happy_run_elastic_fault_tolerant_fails',
                           max_failures=2):
            with tempdir() as dir:
                # these files make training function fail in given rank, epoch and batch
                # we have as many failures as Spark has max_failures (per task / index)
                with open(os.path.sep.join([dir, 'rank_1_epoch_2_batch_4_fail']), 'w'), \
                     open(os.path.sep.join([dir, 'rank_1_epoch_3_batch_1_fail']), 'w'):
                    pass
                res = horovod.spark.run_elastic(
                    fn,
                    args=(2, 5, 5, dir),
                    env={'HOROVOD_LOG_LEVEL': 'DEBUG'},
                    num_proc=2,
                    min_num_proc=2,
                    max_num_proc=2,
                    start_timeout=5,
                    verbose=2)
                self.assertListEqual([([0, 4, 0, 4, 1, 4, 0, 4], 0),
                                      ([0, 4, 0, 4, 1, 4, 0, 4], 1)], res)
Esempio n. 8
0
def single_tiff_setup(n_images, fmt='foo-{:05}.tif'):
    with tempdir() as d:
        data = np.ones((512, 512), dtype=np.float32)

        for i in range(n_images):
            tifffile.imsave(d.path(fmt.format(i)), data)

        yield d
Esempio n. 9
0
def single_tiff_setup(n_images, fmt='foo-{:05}.tif'):
    with tempdir() as d:
        data = np.ones((512, 512), dtype=np.float32)

        for i in range(n_images):
            tifffile.imsave(d.path(fmt.format(i)), data)

        yield d
Esempio n. 10
0
    def test_gen_pipeline_with_empty_changes(self):
        with tempdir() as dir:
            tmp_gen_pipeline_sh = os.path.join(dir, 'gen-pipeline.sh')
            copy('../.buildkite/gen-pipeline.sh', tmp_gen_pipeline_sh)

            with open(os.path.join(dir, 'get_commit_files.py'), 'w') as py:
                py.write("pass")

            self.do_test_gen_full_pipeline(tmp_gen_pipeline_sh)
Esempio n. 11
0
    def test_gen_pipeline_on_default_branch(self):
        with tempdir() as dir:
            tmp_gen_pipeline_sh = os.path.join(dir, 'gen-pipeline.sh')
            copy('../.buildkite/gen-pipeline.sh', tmp_gen_pipeline_sh)

            with open(os.path.join(dir, 'get_commit_files.py'), 'w') as py:
                py.write("print('.github/new_file')")

            env = dict(BUILDKITE_BRANCH='default', BUILDKITE_PIPELINE_DEFAULT_BRANCH='default')
            self.do_test_gen_full_pipeline(tmp_gen_pipeline_sh, env)
Esempio n. 12
0
def test_get_spark_df_input_files(spark):
    with tempdir() as d:
        pq_dir = os.path.join(d, 'test_spark_df_output')
        with spark_session('test_get_spark_df_input_files') as spark:
            spark.range(100).repartition(4).write.parquet(pq_dir)

        pq_files = _get_spark_df_saved_file_list(pq_dir)
        pq_files = sorted(pq_files)
        assert len(pq_files) == 4
        for i in range(4):
            assert pq_files[i].startswith('part-0000' + str(i))
Esempio n. 13
0
    def test_gen_pipeline_with_failing_py(self):
        with tempdir() as dir:
            tmp_gen_pipeline_sh = os.path.join(dir, 'gen-pipeline.sh')
            copy(GEN_PIPELINE_FNAME, tmp_gen_pipeline_sh)

            with open(os.path.join(dir, 'get_changed_code_files.py'),
                      'w') as py:
                py.write('import sys\n')
                py.write('sys.exit(1)')

            self.do_test_gen_full_pipeline(tmp_gen_pipeline_sh)
Esempio n. 14
0
def test_memory_in():
    with tempdir() as d:
        from ufo import MemoryIn, Write

        ref = random.astype(np.float32)
        read = MemoryIn(pointer=ref.__array_interface__['data'][0], number=1,
                        width=ref.shape[1], height=ref.shape[0])
        write = Write(filename=d.path('foo.tif'))

        write(read()).run().join()
        result = tifffile.imread(d.path('foo.tif'))
        assert(np.all(ref == result))
Esempio n. 15
0
    def test_gen_pipeline_on_default_branch(self):
        with tempdir() as dir:
            tmp_gen_pipeline_sh = os.path.join(dir, 'gen-pipeline.sh')
            copy(GEN_PIPELINE_FNAME, tmp_gen_pipeline_sh)

            with open(os.path.join(dir, 'get_changed_code_files.py'),
                      'w') as py:
                py.write("pass")

            env = dict(BUILDKITE_BRANCH='default',
                       BUILDKITE_PIPELINE_DEFAULT_BRANCH='default')
            self.do_test_gen_full_pipeline(tmp_gen_pipeline_sh, env)
Esempio n. 16
0
def test_task_count():
    from ufo import DummyData, Write, Average

    with tempdir() as d:
        generate = DummyData(number=5, width=512, height=512)
        write = Write(filename=d.path('foo-%i.tif'))
        average = Average()

        write(average(generate())).run().join()

        assert (generate.task.props.num_processed == 0)
        assert (average.task.props.num_processed == 5)
        assert (write.task.props.num_processed == 1)
Esempio n. 17
0
def test_memory_out():
    with tempdir() as d:
        from ufo import MemoryOut, Read

        ref = random.astype(np.float32)
        out = np.zeros_like(ref).astype(np.float32)
        tifffile.imsave(d.path('foo.tif'), ref)

        read = Read(path=d.path('foo.tif'))
        write = MemoryOut(pointer=out.__array_interface__['data'][0], max_size=ref.nbytes)

        write(read()).run().join()
        assert(np.all(out == ref))
Esempio n. 18
0
def test_read_multi_tiffs():
    from ufo import Read, Null

    with tempdir() as d:
        n_images = 32
        data = np.ones((512, 512, n_images))
        tifffile.imsave(d.path('foo.tif'), data)

        read = Read(path=d.path('foo.tif'))
        null = Null()

        null(read()).run().join()
        assert (null.task.props.num_processed == n_images)
Esempio n. 19
0
def test_task_count():
    from ufo import DummyData, Write, Average

    with tempdir() as d:
        generate = DummyData(number=5, width=512, height=512)
        write = Write(filename=d.path('foo-%i.tif'))
        average = Average()

        write(average(generate())).run().join()

        assert(generate.task.props.num_processed == 0)
        assert(average.task.props.num_processed == 5)
        assert(write.task.props.num_processed == 1)
Esempio n. 20
0
def test_read_multi_tiffs():
    from ufo import Read, Null

    with tempdir() as d:
        n_images = 32
        data = np.ones((512, 512, n_images))
        tifffile.imsave(d.path('foo.tif'), data)

        read = Read(path=d.path('foo.tif'))
        null = Null()

        null(read()).run().join()
        assert(null.task.props.num_processed == n_images)
Esempio n. 21
0
def generate_go_ethereum_fixture(destination_dir):
    with contextlib.ExitStack() as stack:
        datadir = stack.enter_context(common.tempdir())

        keystore_dir = os.path.join(datadir, 'keystore')
        common.ensure_path_exists(keystore_dir)
        keyfile_path = os.path.join(keystore_dir, common.KEYFILE_FILENAME)
        with open(keyfile_path, 'w') as keyfile:
            keyfile.write(common.KEYFILE_DATA)
        genesis_file_path = os.path.join(datadir, 'genesis.json')
        with open(genesis_file_path, 'w') as genesis_file:
            genesis_file.write(json.dumps(common.GENESIS_DATA))

        geth_ipc_path_dir = stack.enter_context(common.tempdir())
        geth_ipc_path = os.path.join(geth_ipc_path_dir, 'geth.ipc')

        geth_port = get_open_port()
        geth_binary = common.get_geth_binary()

        geth_proc = stack.enter_context(common.get_geth_process(  # noqa: F841
            geth_binary=geth_binary,
            datadir=datadir,
            genesis_file_path=genesis_file_path,
            ipc_path=geth_ipc_path,
            port=geth_port,
            networkid=str(common.GENESIS_DATA['config']['chainId'])
        ))

        common.wait_for_socket(geth_ipc_path)
        web3 = Web3(Web3.IPCProvider(geth_ipc_path))
        chain_data = setup_chain_state(web3)
        static_data = {
            'raw_txn_account': common.RAW_TXN_ACCOUNT,
            'keyfile_pw': common.KEYFILE_PW,
        }
        pprint.pprint(merge(chain_data, static_data))

        shutil.copytree(datadir, destination_dir)
Esempio n. 22
0
def generate_go_ethereum_fixture(destination_dir):
    with contextlib.ExitStack() as stack:
        datadir = stack.enter_context(common.tempdir())

        keystore_dir = os.path.join(datadir, 'keystore')
        common.ensure_path_exists(keystore_dir)
        keyfile_path = os.path.join(keystore_dir, common.KEYFILE_FILENAME)
        with open(keyfile_path, 'w') as keyfile:
            keyfile.write(common.KEYFILE_DATA)
        genesis_file_path = os.path.join(datadir, 'genesis.json')
        with open(genesis_file_path, 'w') as genesis_file:
            genesis_file.write(json.dumps(common.GENESIS_DATA))

        geth_ipc_path_dir = stack.enter_context(common.tempdir())
        geth_ipc_path = os.path.join(geth_ipc_path_dir, 'geth.ipc')

        geth_port = common.get_open_port()
        geth_binary = common.get_geth_binary()

        geth_proc = stack.enter_context(
            common.get_geth_process(  # noqa: F841
                geth_binary=geth_binary,
                datadir=datadir,
                genesis_file_path=genesis_file_path,
                ipc_path=geth_ipc_path,
                port=geth_port,
                networkid=str(common.GENESIS_DATA['config']['chainId'])))

        common.wait_for_socket(geth_ipc_path)
        web3 = Web3(Web3.IPCProvider(geth_ipc_path))
        chain_data = setup_chain_state(web3)
        static_data = {
            'raw_txn_account': common.RAW_TXN_ACCOUNT,
            'keyfile_pw': common.KEYFILE_PW,
        }
        pprint.pprint(merge(chain_data, static_data))

        shutil.copytree(datadir, destination_dir)
Esempio n. 23
0
    def test_gen_pipeline_with_code_changes(self):
        with tempdir() as dir:
            tmp_gen_pipeline_sh = os.path.join(dir, 'gen-pipeline.sh')
            copy('../.buildkite/gen-pipeline.sh', tmp_gen_pipeline_sh)

            for filename in [
                    '.buildkite/gen-pipeline.sh', 'cmake/file',
                    'examples/file', 'horovod/file', 'test/file',
                    'Dockerfile.cpu', ''
            ]:
                with open(os.path.join(dir, 'get_commit_files.py'), 'w') as py:
                    py.write("print('{}')".format(filename))

                self.do_test_gen_full_pipeline(tmp_gen_pipeline_sh)
Esempio n. 24
0
def test_memory_in():
    with tempdir() as d:
        from ufo import MemoryIn, Write

        ref = random.astype(np.float32)
        read = MemoryIn(pointer=ref.__array_interface__['data'][0],
                        number=1,
                        width=ref.shape[1],
                        height=ref.shape[0])
        write = Write(filename=d.path('foo.tif'))

        write(read()).run().join()
        result = tifffile.imread(d.path('foo.tif'))
        assert (np.all(ref == result))
Esempio n. 25
0
def test_memory_out():
    with tempdir() as d:
        from ufo import MemoryOut, Read

        ref = random.astype(np.float32)
        out = np.zeros_like(ref).astype(np.float32)
        tifffile.imsave(d.path('foo.tif'), ref)

        read = Read(path=d.path('foo.tif'))
        write = MemoryOut(pointer=out.__array_interface__['data'][0],
                          max_size=ref.nbytes)

        write(read()).run().join()
        assert (np.all(out == ref))
Esempio n. 26
0
def test_core_issue_61_16_bit_tiffs():
    from ufo import Read, Write

    orig = np.random.randint(0, 65535, (512, 512)).astype(np.uint16)

    with tempdir() as d:
        tifffile.imsave(d.path('temp.tif'), orig)

        read = Read(path=d.path('temp.tif'))
        write = Write(filename=d.path('temp-%i.tif'))

        write(read()).run().join()

        produced = tifffile.imread(d.path('temp-0.tif'))
        assert(np.sum(orig - produced) == 0.0)
Esempio n. 27
0
def test_core_issue_61_16_bit_tiffs():
    from ufo import Read, Write

    orig = np.random.randint(0, 65535, (512, 512)).astype(np.uint16)

    with tempdir() as d:
        tifffile.imsave(d.path('temp.tif'), orig)

        read = Read(path=d.path('temp.tif'))
        write = Write(filename=d.path('temp-%i.tif'))

        write(read()).run().join()

        produced = tifffile.imread(d.path('temp-0.tif'))
        assert (np.sum(orig - produced) == 0.0)
Esempio n. 28
0
    def test_gen_pipeline_with_non_code_changes(self):
        with tempdir() as dir:
            tmp_gen_pipeline_sh = os.path.join(dir, 'gen-pipeline.sh')
            copy(GEN_PIPELINE_FNAME, tmp_gen_pipeline_sh)

            with open(os.path.join(dir, 'get_changed_code_files.py'),
                      'w') as py:
                py.write("pass")

            gen_pipeline_env = dict(BUILDKITE_PIPELINE_SLUG='SLUG',
                                    BUILDKITE_BRANCH='BRANCH')
            exit_code, actual_pipeline, gen_pipeline_log = self._run(
                tmp_gen_pipeline_sh, gen_pipeline_env)

            self.assertEqual(0, exit_code)
            self.assertEqual('', gen_pipeline_log)
            self.assertEqual("steps:\n"
                             "- wait\n"
                             "- wait\n"
                             "- wait\n", actual_pipeline)
Esempio n. 29
0
def test_broadcast():
    from ufo import Generate, Writer
    import glob

    with tempdir() as d:
        generate = Generate(number=5, width=512, height=512)
        write1 = Writer(filename=d.path('foo-%i.tif'))
        write2 = Writer(filename=d.path('bar-%i.tif'))

        g = Ufo.TaskGraph()
        g.connect_nodes(generate.task, write1.task)
        g.connect_nodes(generate.task, write2.task)

        sched = Ufo.Scheduler()
        sched.run(g)

        foos = glob.glob(d.path('foo-*'))
        bars = glob.glob(d.path('bar-*'))
        assert(len(foos) == 5)
        assert(len(bars) == 5)
Esempio n. 30
0
def test_broadcast():
    from ufo import Generate, Writer
    import glob

    with tempdir() as d:
        generate = Generate(number=5, width=512, height=512)
        write1 = Writer(filename=d.path('foo-%i.tif'))
        write2 = Writer(filename=d.path('bar-%i.tif'))

        g = Ufo.TaskGraph()
        g.connect_nodes(generate.task, write1.task)
        g.connect_nodes(generate.task, write2.task)

        sched = Ufo.Scheduler()
        sched.run(g)

        foos = glob.glob(d.path('foo-*'))
        bars = glob.glob(d.path('bar-*'))
        assert (len(foos) == 5)
        assert (len(bars) == 5)
Esempio n. 31
0
def test_wait_file_available_on_dbfs():
    with tempdir() as d:
        pq_dir = os.path.join(d, 'test_ev')
        os.makedirs(pq_dir)
        file1_path = os.path.join(pq_dir, 'file1')
        file2_path = os.path.join(pq_dir, 'file2')
        url1 = 'file://' + file1_path.replace(os.sep, '/')
        url2 = 'file://' + file2_path.replace(os.sep, '/')

        url_list = [url1, url2]

        def create_file(p):
            with open(p, 'w'):
                pass

        # 1. test all files exists.
        create_file(file1_path)
        create_file(file2_path)
        _wait_file_available_on_dbfs(url_list)

        # 2. test one file does not exists. Raise error.
        os.remove(file2_path)
        with pytest.raises(
                RuntimeError,
                match=
                'Timeout while waiting for all parquet-store files to appear'):
            _wait_file_available_on_dbfs(url_list)

        # 3. test one file accessible after 1 second.
        def delay_create_file2():
            time.sleep(1)
            create_file(file2_path)

        threading.Thread(target=delay_create_file2()).start()

        _wait_file_available_on_dbfs(url_list)
Esempio n. 32
0
def generate_parity_fixture(destination_dir):
    """
    The parity fixture generation strategy is to start a ghuc client with
    existing fixtures copied into a temp datadir.  Then a parity client
    is started is peered with the ghuc client.
    """
    with contextlib.ExitStack() as stack:

        ghuc_datadir = stack.enter_context(common.tempdir())

        ghuc_port = common.get_open_port()

        ghuc_ipc_path_dir = stack.enter_context(common.tempdir())
        ghuc_ipc_path = os.path.join(ghuc_ipc_path_dir, 'ghuc.ipc')

        ghuc_keystore_dir = os.path.join(ghuc_datadir, 'keystore')
        common.ensure_path_exists(ghuc_keystore_dir)
        ghuc_keyfile_path = os.path.join(ghuc_keystore_dir,
                                         common.KEYFILE_FILENAME)
        with open(ghuc_keyfile_path, 'w') as keyfile:
            keyfile.write(common.KEYFILE_DATA)

        genesis_file_path = os.path.join(ghuc_datadir, 'genesis.json')
        with open(genesis_file_path, 'w') as genesis_file:
            genesis_file.write(json.dumps(common.GENESIS_DATA))

        stack.enter_context(
            common.get_ghuc_process(common.get_ghuc_binary(), ghuc_datadir,
                                    genesis_file_path, ghuc_ipc_path,
                                    ghuc_port,
                                    str(CHAIN_CONFIG['params']['networkID'])))
        # set up fixtures
        common.wait_for_socket(ghuc_ipc_path)
        webu_ghuc = Webu(Webu.IPCProvider(ghuc_ipc_path))
        chain_data = go_happyuc.setup_chain_state(webu_ghuc)
        fixture_block_count = webu_ghuc.eth.blockNumber

        datadir = stack.enter_context(common.tempdir())

        keystore_dir = os.path.join(datadir, 'keys')
        os.makedirs(keystore_dir, exist_ok=True)
        parity_keyfile_path = os.path.join(keystore_dir,
                                           common.KEYFILE_FILENAME)
        with open(parity_keyfile_path, 'w') as keyfile:
            keyfile.write(common.KEYFILE_DATA)

        chain_config_file_path = os.path.join(datadir, 'chain_config.json')
        with open(chain_config_file_path, 'w') as chain_file:
            chain_file.write(json.dumps(CHAIN_CONFIG))

        parity_ipc_path_dir = stack.enter_context(common.tempdir())
        parity_ipc_path = os.path.join(parity_ipc_path_dir, 'jsonrpc.ipc')

        parity_port = common.get_open_port()
        parity_binary = get_parity_binary()

        parity_proc = stack.enter_context(
            get_parity_process(  # noqa: F841
                parity_binary=parity_binary,
                datadir=datadir,
                ipc_path=parity_ipc_path,
                keys_path=keystore_dir,
                chain_config_file_path=chain_config_file_path,
                parity_port=parity_port,
            ))

        common.wait_for_socket(parity_ipc_path)
        webu = Webu(Webu.IPCProvider(parity_ipc_path))

        time.sleep(10)
        connect_nodes(webu, webu_ghuc)
        wait_for_chain_sync(webu, fixture_block_count)

        static_data = {
            'raw_txn_account': common.RAW_TXN_ACCOUNT,
            'keyfile_pw': common.KEYFILE_PW,
        }
        pprint.pprint(merge(chain_data, static_data))

        shutil.copytree(datadir, destination_dir)

        parity_proc = stack.enter_context(
            parity_export_blocks_process(  # noqa: F841
                parity_binary=parity_binary,
                datadir=destination_dir,
                chain_config_file_path=os.path.join(destination_dir,
                                                    'chain_config.json'),
                parity_port=parity_port,
            ))
Esempio n. 33
0
def local_store():
    with tempdir() as tmp:
        store = LocalStore(tmp)
        yield store
Esempio n. 34
0
def generate_parity_fixture(destination_dir):
    """
    The parity fixture generation strategy is to start a geth client with
    existing fixtures copied into a temp datadir.  Then a parity client
    is started is peered with the geth client.
    """
    with contextlib.ExitStack() as stack:

        geth_datadir = stack.enter_context(common.tempdir())

        geth_port = get_open_port()

        geth_ipc_path_dir = stack.enter_context(common.tempdir())
        geth_ipc_path = os.path.join(geth_ipc_path_dir, 'geth.ipc')

        geth_keystore_dir = os.path.join(geth_datadir, 'keystore')
        common.ensure_path_exists(geth_keystore_dir)
        geth_keyfile_path = os.path.join(geth_keystore_dir, common.KEYFILE_FILENAME)
        with open(geth_keyfile_path, 'w') as keyfile:
            keyfile.write(common.KEYFILE_DATA)

        genesis_file_path = os.path.join(geth_datadir, 'genesis.json')
        with open(genesis_file_path, 'w') as genesis_file:
            genesis_file.write(json.dumps(common.GENESIS_DATA))

        stack.enter_context(
            common.get_geth_process(
                common.get_geth_binary(),
                geth_datadir,
                genesis_file_path,
                geth_ipc_path,
                geth_port,
                str(CHAIN_CONFIG['params']['networkID']))
        )
        # set up fixtures
        common.wait_for_socket(geth_ipc_path)
        web3_geth = Web3(Web3.IPCProvider(geth_ipc_path))
        chain_data = go_ethereum.setup_chain_state(web3_geth)
        fixture_block_count = web3_geth.eth.blockNumber

        datadir = stack.enter_context(common.tempdir())

        keystore_dir = os.path.join(datadir, 'keys')
        os.makedirs(keystore_dir, exist_ok=True)
        parity_keyfile_path = os.path.join(keystore_dir, common.KEYFILE_FILENAME)
        with open(parity_keyfile_path, 'w') as keyfile:
            keyfile.write(common.KEYFILE_DATA)

        chain_config_file_path = os.path.join(datadir, 'chain_config.json')
        with open(chain_config_file_path, 'w') as chain_file:
            chain_file.write(json.dumps(CHAIN_CONFIG))

        parity_ipc_path_dir = stack.enter_context(common.tempdir())
        parity_ipc_path = os.path.join(parity_ipc_path_dir, 'jsonrpc.ipc')

        parity_port = get_open_port()
        parity_binary = get_parity_binary()

        parity_proc = stack.enter_context(get_parity_process(  # noqa: F841
            parity_binary=parity_binary,
            datadir=datadir,
            ipc_path=parity_ipc_path,
            keys_path=keystore_dir,
            chain_config_file_path=chain_config_file_path,
            parity_port=parity_port,
        ))

        common.wait_for_socket(parity_ipc_path)
        web3 = Web3(Web3.IPCProvider(parity_ipc_path))

        time.sleep(10)
        connect_nodes(web3, web3_geth)
        wait_for_chain_sync(web3, fixture_block_count)

        static_data = {
            'raw_txn_account': common.RAW_TXN_ACCOUNT,
            'keyfile_pw': common.KEYFILE_PW,
        }
        pprint.pprint(merge(chain_data, static_data))

        shutil.copytree(datadir, destination_dir)

        parity_proc = stack.enter_context(parity_export_blocks_process(  # noqa: F841
            parity_binary=parity_binary,
            datadir=destination_dir,
            chain_config_file_path=os.path.join(destination_dir, 'chain_config.json'),
            parity_port=parity_port,
        ))