def test_dickens(mrs_impl, mrs_reduce_tasks, tmpdir):
inputs = glob.glob('tests/data/dickens/*')
outdir = tmpdir.join('out')
with tempfile.NamedTemporaryFile('wt') as listfile:
for filename in inputs:
print(filename, file=listfile)
listfile.flush()
args = [listfile.name, outdir.strpath]
if mrs_impl == 'serial':
assert mrs_reduce_tasks == 1
run_serial(WordCount2, args)
elif mrs_impl == 'mockparallel':
args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
run_mockparallel(WordCount2, args, tmpdir)
elif mrs_impl == 'master_slave':
args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
run_master_slave(WordCount2, args, tmpdir)
else:
raise RuntimeError('Unknown mrs_impl: %s' % mrs_impl)
files = outdir.listdir()
assert len(files) == mrs_reduce_tasks
counts = defaultdict(int)
for outfile in files:
text = outfile.readlines()
last_key = None
for line in text:
key, value = line.split()
counts[key] += int(value)
assert last_key is None or last_key <= key
last_key = key
# Check counts for all of the words in the first two lines.
assert counts['it'] == 11
assert counts['was'] == 12
assert counts['the'] == 18
assert counts['best'] == 1
assert counts['of'] == 16
assert counts['worst'] == 1
assert counts['times'] == 2
# Check counts for several other miscellaneous words.
assert counts['a'] == 8
assert counts['heaven'] == 1
assert counts['period'] == 2
assert counts['settled'] == 1
assert counts['for'] == 3
def test_dickens(mrs_impl, mrs_reduce_tasks, tmpdir):
inputs = glob.glob('tests/data/dickens/*')
outdir = tmpdir.join('out')
args = inputs + [outdir.strpath]
if mrs_impl == 'serial':
assert mrs_reduce_tasks == 1
run_serial(WordCount, args)
elif mrs_impl == 'mockparallel':
args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
run_mockparallel(WordCount, args, tmpdir)
elif mrs_impl == 'master_slave':
args = ['--mrs-reduce-tasks', str(mrs_reduce_tasks)] + args
run_master_slave(WordCount, args, tmpdir)
else:
raise RuntimeError('Unknown mrs_impl: %s' % mrs_impl)
# Make sure no extra temporary files are left behind.
assert len(tmpdir.listdir()) == 1
files = outdir.listdir()
assert len(files) == mrs_reduce_tasks
counts = defaultdict(int)
for outfile in files:
text = outfile.readlines()
last_key = None
for line in text:
key, value = line.split()
counts[key] += int(value)
assert last_key is None or last_key <= key
last_key = key
# Check counts for all of the words in the first two lines.
assert counts['it'] == 11
assert counts['was'] == 12
assert counts['the'] == 18
assert counts['best'] == 1
assert counts['of'] == 16
assert counts['worst'] == 1
assert counts['times'] == 2
# Check counts for several other miscellaneous words.
assert counts['a'] == 8
assert counts['heaven'] == 1
assert counts['period'] == 2
assert counts['settled'] == 1
assert counts['for'] == 3
def test_pso(mrs_impl, tmpdir, capfd):
args = ['-i', '20', '-n', '5', '-d', '2', '--out-freq', '3', '--mrs-seed',
'42', '--hey-im-testing']
if mrs_impl == 'serial':
run_serial(StandardPSO, args)
elif mrs_impl == 'mockparallel':
run_mockparallel(StandardPSO, args, tmpdir)
elif mrs_impl == 'master_slave':
run_master_slave(StandardPSO, args, tmpdir)
else:
raise RuntimeError('Unknown mrs_impl: %s' % mrs_impl)
out, err = capfd.readouterr()
assert err == ''
lines = [line.strip() for line in out.splitlines()
if line and not line.startswith('#')]
assert lines == ['202.330512851', '202.330512851', '74.0008930067',
'32.3155678366', '2.9191713839', '2.9191713839', '2.9191713839']