def test_sample_matches_score_counts(Model, EXAMPLE, sample_count):
for size in iter_valid_sizes(EXAMPLE, max_size=10):
model = Model()
model.load(EXAMPLE)
samples = []
probs_dict = {}
for _ in xrange(sample_count):
value = model.sample_assignments(size)
sample = canonicalize(value)
samples.append(sample)
if sample not in probs_dict:
assignments = dict(enumerate(value))
counts = count_assignments(assignments)
prob = math.exp(model.score_counts(counts))
probs_dict[sample] = prob
# renormalize here; test normalization separately
total = sum(probs_dict.values())
for key in probs_dict:
probs_dict[key] /= total
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
print '{} gof = {:0.3g}'.format(Model.__name__, gof)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def _check_marginal_samples_match_scores(server, row, fi):
row = loom.query.protobuf_to_data_row(row.diff)
row[fi] = None
to_sample = [i == fi for i in range(len(row))]
samples = server.sample(to_sample, row, SAMPLE_COUNT)
val = samples[0][fi]
base_score = server.score(row)
if isinstance(val, bool) or isinstance(val, int):
probs_dict = {}
samples = [sample[fi] for sample in samples]
for sample in set(samples):
row[fi] = sample
probs_dict[sample] = numpy.exp(
server.score(row) - base_score)
if len(probs_dict) == 1:
assert_almost_equal(probs_dict[sample], 1., places=SCORE_PLACES)
return
if min(probs_dict.values()) < MIN_CATEGORICAL_PROB:
return
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
elif isinstance(val, float):
probs = numpy.exp([
server.score(sample) - base_score
for sample in samples
])
samples = [sample[fi] for sample in samples]
gof = density_goodness_of_fit(samples, probs, plot=True)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def test_sample_value(module, EXAMPLE):
seed_all(0)
shared = module.Shared.from_dict(EXAMPLE['shared'])
shared.realize()
for values in [[], EXAMPLE['values']]:
group = module.Group.from_values(shared, values)
sample_count = SAMPLE_COUNT
if module.Value == numpy.ndarray:
sample_count *= 10
samples = [group.sample_value(shared) for _ in xrange(sample_count)]
if module.Value in [bool, int]:
probs_dict = {
value: math.exp(group.score_value(shared, value))
for value in set(samples)
}
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
elif module.Value == float:
probs = numpy.exp(
[group.score_value(shared, value) for value in samples])
gof = density_goodness_of_fit(samples, probs, plot=True)
elif module.Value == numpy.ndarray:
if module.__name__ == 'distributions.lp.models.niw':
raise SkipTest('FIXME known sampling bug')
probs = numpy.exp(
[group.score_value(shared, value) for value in samples])
gof = vector_density_goodness_of_fit(samples, probs, plot=True)
else:
raise SkipTest('Not implemented for {}'.format(module.Value))
print '{} gof = {:0.3g}'.format(module.__name__, gof)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def _test_scipy_stats(name):
if name in known_failures:
raise SkipTest('known failure')
dist = getattr(scipy.stats, name)
try:
params = default_params[name]
except KeyError:
params = [tuple(1.0 + rand(dist.numargs))]
for param in params:
print 'param = {}'.format(param)
dim = get_dim(dist.rvs(*param))
sample_count = 100 + 1000 * dim
samples = list(dist.rvs(*param, size=sample_count))
if hasattr(dist, 'pmf'):
probs = [dist.pmf(sample, *param) for sample in samples]
probs_dict = dict(izip(samples, probs))
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
else:
probs = [dist.pdf(sample, *param) for sample in samples]
gof = auto_density_goodness_of_fit(samples, probs, plot=True)
assert_greater(gof, TEST_FAILURE_RATE)
gof = mixed_density_goodness_of_fit(samples, probs, plot=True)
assert_greater(gof, TEST_FAILURE_RATE)
def _check_marginal_samples_match_scores(server, row, fi):
row = loom.query.protobuf_to_data_row(row.diff)
row[fi] = None
to_sample = [i == fi for i in range(len(row))]
samples = server.sample(to_sample, row, SAMPLE_COUNT)
val = samples[0][fi]
base_score = server.score(row)
if isinstance(val, bool) or isinstance(val, (int, long)):
probs_dict = {}
samples = [sample[fi] for sample in samples]
for sample in set(samples):
row[fi] = sample
probs_dict[sample] = numpy.exp(server.score(row) - base_score)
if len(probs_dict) == 1:
assert_almost_equal(probs_dict[sample], 1., places=SCORE_PLACES)
return
if min(probs_dict.values()) < MIN_CATEGORICAL_PROB:
return
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
elif isinstance(val, float):
probs = numpy.exp(
[server.score(sample) - base_score for sample in samples])
samples = [sample[fi] for sample in samples]
gof = density_goodness_of_fit(samples, probs, plot=True)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def test_sample_matches_score_counts(Model, EXAMPLE, sample_count):
for size in iter_valid_sizes(EXAMPLE, max_size=10):
model = Model()
model.load(EXAMPLE)
samples = []
probs_dict = {}
for _ in xrange(sample_count):
value = model.sample_assignments(size)
sample = canonicalize(value)
samples.append(sample)
if sample not in probs_dict:
assignments = dict(enumerate(value))
counts = count_assignments(assignments)
prob = math.exp(model.score_counts(counts))
probs_dict[sample] = prob
# renormalize here; test normalization separately
total = sum(probs_dict.values())
for key in probs_dict:
probs_dict[key] /= total
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
print '{} gof = {:0.3g}'.format(Model.__name__, gof)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def test_sample_value(module, EXAMPLE):
seed_all(0)
shared = module.Shared.from_dict(EXAMPLE['shared'])
shared.realize()
for values in [[], EXAMPLE['values']]:
group = module.Group.from_values(shared, values)
sample_count = SAMPLE_COUNT
if module.Value == numpy.ndarray:
sample_count *= 10
samples = [group.sample_value(shared) for _ in xrange(sample_count)]
if module.Value in [bool, int]:
probs_dict = {
value: math.exp(group.score_value(shared, value))
for value in set(samples)
}
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
elif module.Value == float:
probs = numpy.exp([
group.score_value(shared, value)
for value in samples
])
gof = density_goodness_of_fit(samples, probs, plot=True)
elif module.Value == numpy.ndarray:
if module.__name__ == 'distributions.lp.models.niw':
raise SkipTest('FIXME known sampling bug')
probs = numpy.exp([
group.score_value(shared, value)
for value in samples
])
gof = vector_density_goodness_of_fit(samples, probs, plot=True)
else:
raise SkipTest('Not implemented for {}'.format(module.Value))
print '{} gof = {:0.3g}'.format(module.__name__, gof)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def test_sample_group(module, EXAMPLE):
seed_all(0)
SIZE = 2
shared = module.Shared.from_dict(EXAMPLE['shared'])
shared.realize()
for values in [[], EXAMPLE['values']]:
if module.Value in [bool, int]:
samples = []
probs_dict = {}
for _ in xrange(SAMPLE_COUNT):
values = module.sample_group(shared, SIZE)
sample = tuple(values)
samples.append(sample)
group = module.Group.from_values(shared, values)
probs_dict[sample] = math.exp(group.score_data(shared))
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
else:
raise SkipTest('Not implemented for {}'.format(module.Value))
print '{} gof = {:0.3g}'.format(module.__name__, gof)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def test_sample_group(module, EXAMPLE):
seed_all(0)
SIZE = 2
shared = module.Shared.from_dict(EXAMPLE['shared'])
shared.realize()
for values in [[], EXAMPLE['values']]:
if module.Value in [bool, int]:
samples = []
probs_dict = {}
for _ in xrange(SAMPLE_COUNT):
values = module.sample_group(shared, SIZE)
sample = tuple(values)
samples.append(sample)
group = module.Group.from_values(shared, values)
probs_dict[sample] = math.exp(group.score_data(shared))
gof = discrete_goodness_of_fit(samples, probs_dict, plot=True)
else:
raise SkipTest('Not implemented for {}'.format(module.Value))
print '{} gof = {:0.3g}'.format(module.__name__, gof)
assert_greater(gof, MIN_GOODNESS_OF_FIT)
def goodness_of_fit(self, samples, probabilities):
probs_dict = dict(zip(samples, probabilities))
gof = discrete_goodness_of_fit(samples, probs_dict)
return gof
def goodness_of_fit(self, samples, probabilities):
probs_dict = dict(zip(samples, probabilities))
gof = discrete_goodness_of_fit(samples, probs_dict)
return gof
