def main():
"""The selected databases/kwargs/event found for different algorithms
| Algorithm | Databases | kwargs | Event |
|:-------------:|:---------:|:------:|:-----:|
| noisy_max_v1a | | | |
| noisy_max_v1b | | | |
| noisy_max_v2a | | | |
| noisy_max_v2b | | | |
| histogram | | | |
| histogram_eps | | | |
| SVT | | | |
| iSVT1 | | | |
| iSVT2 | | | |
| iSVT3 | | | |
| iSVT4 | | | |
"""
tasks = [(noisy_max_v1a, {}), (noisy_max_v1b, {}), (noisy_max_v2a, {}),
(noisy_max_v2b, {}), (histogram, {}), (histogram_eps, {}),
(SVT, {
'N': 1,
'T': 0.5
}), (iSVT1, {
'T': 1,
'N': 1
}), (iSVT2, {
'T': 1,
'N': 1
}), (iSVT3, {
'T': 1,
'N': 1
}), (iSVT4, {
'T': 1,
'N': 1
})]
for i, (algorithm, kwargs) in enumerate(tasks):
start_time = time.time()
results = {}
for privacy_budget in (0.2, 0.7, 1.5):
kwargs['epsilon'] = privacy_budget
sensitivity = ONE_DIFFER if 'histogram' in algorithm.__name__ else ALL_DIFFER
results[privacy_budget] = detect_counterexample(
algorithm,
tuple(x / 10.0 for x in range(1, 34, 1)),
kwargs,
sensitivity=sensitivity)
plot_result(r'Test $\epsilon$', 'P Value', results,
algorithm.__name__.replace('_', ' ').title(),
algorithm.__name__ + '.pdf')
# dump the results to file
with open('./{}.json'.format(algorithm.__name__), 'w') as f:
json.dump(results, f)
logger.info('[{} / {}]: {} | Time elapsed: {}'.format(
i + 1, len(tasks), algorithm.__name__,
time.time() - start_time))
def test_incorrect_algorithm(algorithm):
func, kwargs, num_input, sensitivity = algorithm
kwargs.update({'epsilon': 0.7})
result = detect_counterexample(func,
0.7,
kwargs,
num_input=num_input,
loglevel=logging.DEBUG,
sensitivity=sensitivity)
assert isinstance(result, list) and len(result) == 1
epsilon, p, *extras = result[0]
assert p <= 0.05, 'epsilon: {}, p-value: {} is not expected. extra info: {}'.format(
epsilon, p, extras)
def assert_incorrect_algorithm(algorithm, kwargs=None, num_input=5):
if kwargs and isinstance(kwargs, dict):
kwargs.update({'epsilon': 0.7})
else:
kwargs = {'epsilon': 0.7}
result = detect_counterexample(algorithm,
0.7,
kwargs,
num_input=num_input,
loglevel=logging.DEBUG)
assert isinstance(result, list) and len(result) == 1
epsilon, p, *extras = result[0]
assert p <= 0.05, 'epsilon: {}, p-value: {} is not expected. extra info: {}'.format(
epsilon, p, extras)
def test_large_iterations():
result = detect_counterexample(SVT,
0.7, {
'T': 0.5,
'N': 1,
'epsilon': 0.7
},
num_input=10,
loglevel=logging.DEBUG,
sensitivity=ALL_DIFFER,
event_iterations=int(2e6),
detect_iterations=int(5e6))
epsilon, p, *extras = result[0]
assert p >= 0.05, 'epsilon: {}, p-value: {} is not expected. extra info: {}'.format(
epsilon, p, extras)