def test_close(self, argument, shape, type, gen_len, ref_len, use_tqdm):
# 'default' or 'custom'
# 'pad' or 'jag'
# 'list' or 'array'
# 'equal' or 'unequal'
# 'random', 'non-empty', 'empty'
# 'random', 'non-empty', 'empty'
if use_tqdm:
sample = random.randint(1000, 2000)
assert sample >= 1000
else:
sample = random.randint(2, 50)
assert 1 < sample < 1000
fake_bleu_irl_fw = [random.random() for _ in range(sample)]
fake_bleu_irl_bw = [random.random() for _ in range(sample)]
def get_fake_values(lists=(fake_bleu_irl_fw, fake_bleu_irl_bw), i=-1):
def _get_fake_values(*_, **__):
nonlocal i, lists
i = (1 + i) % len(lists)
return lists[i]
return _get_fake_values
import multiprocessing
with mock.patch('tqdm.tqdm', side_effect=get_fake_values()):
with mock.patch('multiprocessing.pool.Pool'):
# tqdm.tqdm is replaced by _get_fake_values. It returns fake values.
dataloader = FakeDataLoader()
reference_key, gen_key = ('resp_allvocabs', 'gen') \
if argument == 'default' else ('rk', 'gk')
data = dataloader.get_data(reference_key=reference_key, gen_key=gen_key, \
to_list=(type == 'list'), pad=(shape == 'pad'), \
gen_len=gen_len, ref_len=ref_len, batch=sample)
# dataloader.data["test"][reference_key] = data[reference_key]
_data = copy.deepcopy(data)
if argument == 'default':
bcm = FwBwBleuCorpusMetric(dataloader, data[reference_key], sample=sample)
else:
bcm = FwBwBleuCorpusMetric(dataloader, data[reference_key], gen_key, sample=sample)
rng_state_st = random.getstate()
assert bcm.sample == sample
bcm.forward(data)
if use_tqdm:
fw_bleu = (1.0 * sum(fake_bleu_irl_fw) / len(fake_bleu_irl_fw))
bw_bleu = (1.0 * sum(fake_bleu_irl_bw) / len(fake_bleu_irl_bw))
if fw_bleu + bw_bleu > 0:
fw_bw_bleu = 2.0 * bw_bleu * fw_bleu / (fw_bleu + bw_bleu)
else:
fw_bw_bleu = 0
assert np.isclose(bcm.close()['fw-bw-bleu'], fw_bw_bleu)
assert tqdm.tqdm.called
assert tqdm.tqdm.call_count == 2
assert tqdm.tqdm() == fake_bleu_irl_fw
assert tqdm.tqdm() == fake_bleu_irl_bw
if bcm.cpu_count > 1:
assert multiprocessing.pool.Pool.called
else:
assert np.isclose(bcm.close()['fw-bw-bleu'],
self.get_bleu(dataloader, _data, reference_key, gen_key))
assert not tqdm.tqdm.called
assert not multiprocessing.pool.Pool.called
assert same_dict(data, _data)
rng_state_ed = random.getstate()
assert operator.eq(rng_state_st, rng_state_ed)
def test_hashvalue(self, to_list, pad):
dataloader = FakeDataLoader()
reference_key, gen_key = ('resp_allvocabs', 'gen')
key_list = [reference_key, gen_key]
data = dataloader.get_data(reference_key=reference_key, gen_key=gen_key, \
to_list=to_list, pad=pad, \
gen_len='non-empty', ref_len='non-empty')
# dataloader.data["test"][reference_key] = data[reference_key]
bcm = FwBwBleuCorpusMetric(dataloader, data[reference_key])
bcm.forward(data)
res = bcm.close()
data_shuffle = shuffle_instances(data, key_list)
# dataloader.data["test"][reference_key] = data_shuffle[reference_key]
bcm_shuffle = FwBwBleuCorpusMetric(dataloader, data_shuffle[reference_key])
bcm_shuffle.forward(data_shuffle)
res_shuffle = bcm_shuffle.close()
assert same_dict(res, res_shuffle, False)
for data_unequal in generate_unequal_data(data, key_list, dataloader.pad_id, \
reference_key, reference_is_3D=False):
# dataloader.data["test"][reference_key] = data_unequal[reference_key]
bcm_unequal = FwBwBleuCorpusMetric(dataloader, data_unequal[reference_key])
bcm_unequal.forward(data_unequal)
res_unequal = bcm_unequal.close()
assert res['fw-bw-bleu hashvalue'] != res_unequal['fw-bw-bleu hashvalue']
bcm_unequal = FwBwBleuCorpusMetric(dataloader, data[reference_key], sample=2)
bcm_unequal.forward(data)
res_unequal = bcm_unequal.close()
assert res['fw-bw-bleu hashvalue'] != res_unequal['fw-bw-bleu hashvalue']