def test_dickens(mrs_impl, mrs_reduce_tasks, tmpdir):
    inputs = glob.glob('tests/data/dickens/*')
    outdir = tmpdir.join('out')
    with tempfile.NamedTemporaryFile('wt') as listfile:
        for filename in inputs:
            print(filename, file=listfile)
        listfile.flush()

        args = [listfile.name, outdir.strpath]

        if mrs_impl == 'serial':
            assert mrs_reduce_tasks == 1
            run_serial(WordCount2, args)
        elif mrs_impl == 'mockparallel':
            args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
            run_mockparallel(WordCount2, args, tmpdir)
        elif mrs_impl == 'master_slave':
            args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
            run_master_slave(WordCount2, args, tmpdir)
        else:
            raise RuntimeError('Unknown mrs_impl: %s' % mrs_impl)

    files = outdir.listdir()
    assert len(files) == mrs_reduce_tasks

    counts = defaultdict(int)
    for outfile in files:
        text = outfile.readlines()
        last_key = None
        for line in text:
            key, value = line.split()
            counts[key] += int(value)
            assert last_key is None or last_key <= key
            last_key = key

    # Check counts for all of the words in the first two lines.
    assert counts['it'] == 11
    assert counts['was'] == 12
    assert counts['the'] == 18
    assert counts['best'] == 1
    assert counts['of'] == 16
    assert counts['worst'] == 1
    assert counts['times'] == 2

    # Check counts for several other miscellaneous words.
    assert counts['a'] == 8
    assert counts['heaven'] == 1
    assert counts['period'] == 2
    assert counts['settled'] == 1
    assert counts['for'] == 3
Example #2
0
def test_dickens(mrs_impl, mrs_reduce_tasks, tmpdir):
    inputs = glob.glob('tests/data/dickens/*')
    outdir = tmpdir.join('out')
    args = inputs + [outdir.strpath]

    if mrs_impl == 'serial':
        assert mrs_reduce_tasks == 1
        run_serial(WordCount, args)
    elif mrs_impl == 'mockparallel':
        args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
        run_mockparallel(WordCount, args, tmpdir)
    elif mrs_impl == 'master_slave':
        args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
        run_master_slave(WordCount, args, tmpdir)
    else:
        raise RuntimeError('Unknown mrs_impl: %s' % mrs_impl)

    # Make sure no extra temporary files are left behind.
    assert len(tmpdir.listdir()) == 1

    files = outdir.listdir()
    assert len(files) == mrs_reduce_tasks

    counts = defaultdict(int)
    for outfile in files:
        text = outfile.readlines()
        last_key = None
        for line in text:
            key, value = line.split()
            counts[key] += int(value)
            assert last_key is None or last_key <= key
            last_key = key

    # Check counts for all of the words in the first two lines.
    assert counts['it'] == 11
    assert counts['was'] == 12
    assert counts['the'] == 18
    assert counts['best'] == 1
    assert counts['of'] == 16
    assert counts['worst'] == 1
    assert counts['times'] == 2

    # Check counts for several other miscellaneous words.
    assert counts['a'] == 8
    assert counts['heaven'] == 1
    assert counts['period'] == 2
    assert counts['settled'] == 1
    assert counts['for'] == 3
Example #3
0
def test_pso(mrs_impl, tmpdir, capfd):
    args = ['-i', '20', '-n', '5', '-d', '2', '--out-freq', '3', '--mrs-seed',
            '42', '--hey-im-testing']

    if mrs_impl == 'serial':
        run_serial(StandardPSO, args)
    elif mrs_impl == 'mockparallel':
        run_mockparallel(StandardPSO, args, tmpdir)
    elif mrs_impl == 'master_slave':
        run_master_slave(StandardPSO, args, tmpdir)
    else:
        raise RuntimeError('Unknown mrs_impl: %s' % mrs_impl)

    out, err = capfd.readouterr()
    assert err == ''

    lines = [line.strip() for line in out.splitlines()
            if line and not line.startswith('#')]
    assert lines == ['202.330512851', '202.330512851', '74.0008930067',
            '32.3155678366', '2.9191713839', '2.9191713839', '2.9191713839']