def thread_test():
np.random.seed(3)
n_moments = 5
distr = stats.norm(loc=1, scale=2)
step_range = [0.01, 0.001, 0.0001]
os.chdir(os.path.dirname(os.path.realpath(__file__)))
work_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'_test_tmp')
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
shutil.copyfile('synth_sim_config.yaml',
os.path.join(work_dir, 'synth_sim_config.yaml'))
simulation_config = {
"config_yaml": os.path.join(work_dir, 'synth_sim_config.yaml')
}
simulation_factory = SynthSimulationWorkspace(simulation_config)
sample_storage = Memory()
sampling_pool = ThreadPool(4, work_dir=work_dir)
# Plan and compute samples
sampler = Sampler(sample_storage=sample_storage,
sampling_pool=sampling_pool,
sim_factory=simulation_factory,
step_range=step_range)
true_domain = distr.ppf([0.0001, 0.9999])
moments_fn = Legendre(n_moments, true_domain)
sampler.set_initial_n_samples()
# sampler.set_initial_n_samples([1000])
sampler.schedule_samples()
sampler.ask_sampling_pool_for_samples()
sampler.target_var_adding_samples(1e-4, moments_fn, sleep=20)
print("collected samples ", sampler._n_created_samples)
means, vars = sampler.estimate_moments(moments_fn)
print("means ", means)
print("vars ", vars)
assert means[0] == 1
assert np.isclose(means[1], 0, atol=1e-2)
assert vars[0] == 0
sampler.schedule_samples()
sampler.ask_sampling_pool_for_samples()
storage = sampler.sample_storage
results = storage.sample_pairs()
def test_storage(storage, n_levels):
if storage == 'memory':
storage = Memory()
elif storage == 'hdf':
work_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'_test_tmp')
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
storage = SampleStorageHDF(
file_path=os.path.join(work_dir, "mlmc.hdf5".format()))
n_successful = 5
n_failed = 4
res_length = 3
format_quant = add_samples(storage,
n_levels,
n_successful=n_successful,
n_failed=n_failed,
res_lenght=res_length)
scheduled = storage.load_scheduled_samples()
assert len(scheduled) == n_levels
for _, l_sch in scheduled.items():
assert len(l_sch) == n_successful + n_failed
results = storage.sample_pairs()
assert len(results) == n_levels
for level_res in results:
assert level_res.shape[1] == n_successful
assert level_res.shape[0] == res_length
assert np.allclose(level_res[:, :, 0], 1)
n_ops = storage.get_n_ops()
assert len(n_ops) == n_levels
loaded_format = storage.load_result_format()
assert len(format_quant) == len(loaded_format)
for f1, f2 in zip(format_quant, loaded_format):
assert f1.name == f2.name
assert f1.unit == f2.unit
n_finished = storage.n_finished()
assert len(n_finished) == n_levels
assert np.allclose(n_finished, n_successful + n_failed)
def _create_sampler(self, step_range, clean=False, memory=False):
# Set work dir
os.chdir(os.path.dirname(os.path.realpath(__file__)))
work_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'_test_tmp')
if clean:
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
# Create simulations
failed_fraction = 0.1
distr = stats.norm()
simulation_config = dict(distr=distr,
complexity=2,
nan_fraction=failed_fraction,
sim_method='_sample_fn')
simulation_factory = SynthSimulationForTests(simulation_config)
# shutil.copyfile('synth_sim_config.yaml', os.path.join(work_dir, 'synth_sim_config.yaml'))
# simulation_config = {"config_yaml": os.path.join(work_dir, 'synth_sim_config.yaml')}
# simulation_workspace = SynthSimulationWorkspace(simulation_config)
# Create sample storages
if memory:
sample_storage = Memory()
else:
sample_storage = SampleStorageHDF(
file_path=os.path.join(work_dir, "mlmc_test.hdf5"))
# Create sampling pools
sampling_pool = OneProcessPool()
# sampling_pool_dir = OneProcessPool(work_dir=work_dir)
if clean:
if sampling_pool._output_dir is not None:
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
if simulation_factory.need_workspace:
os.chdir(os.path.dirname(os.path.realpath(__file__)))
shutil.copyfile(
'synth_sim_config.yaml',
os.path.join(work_dir, 'synth_sim_config.yaml'))
sampler = Sampler(sample_storage=sample_storage,
sampling_pool=sampling_pool,
sim_factory=simulation_factory,
level_parameters=step_range)
return sampler, simulation_factory
def test_sampler():
# Create simulations
failed_fraction = 0.1
distr = stats.norm()
simulation_config = dict(distr=distr,
complexity=2,
nan_fraction=failed_fraction,
sim_method='_sample_fn')
simulation = SynthSimulation(simulation_config)
storage = Memory()
sampling_pool = OneProcessPool()
step_range = [[0.1], [0.01], [0.001]]
sampler = Sampler(sample_storage=storage,
sampling_pool=sampling_pool,
sim_factory=simulation,
level_parameters=step_range)
assert len(sampler._level_sim_objects) == len(step_range)
for step, level_sim in zip(step_range, sampler._level_sim_objects):
assert step[0] == level_sim.config_dict['fine']['step']
init_samples = list(np.ones(len(step_range)) * 10)
sampler.set_initial_n_samples(init_samples)
assert np.allclose(sampler._n_target_samples, init_samples)
assert 0 == sampler.ask_sampling_pool_for_samples()
sampler.schedule_samples()
assert np.allclose(sampler._n_scheduled_samples, init_samples)
n_estimated = np.array([100, 50, 20])
sampler.process_adding_samples(n_estimated, 0, 0.1)
assert np.allclose(sampler._n_target_samples,
init_samples + (n_estimated * 0.1),
atol=1)
shutil.copyfile('synth_sim_config.yaml', os.path.join(work_dir, 'synth_sim_config.yaml'))
simulation_config_workspace = {"config_yaml": os.path.join(work_dir, 'synth_sim_config.yaml')}
def hdf_storage_factory():
os.chdir(os.path.dirname(os.path.realpath(__file__)))
work_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '_test_tmp')
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
# Create sample storages
return SampleStorageHDF(file_path=os.path.join(work_dir, "mlmc_test.hdf5"))
@pytest.mark.parametrize("test_case", [(SynthSimulationForTests(simulation_config), Memory(), OneProcessPool()),
(SynthSimulationForTests(simulation_config), Memory(), ProcessPool(4)),
# (SynthSimulationForTests(simulation_config), Memory(), ThreadPool(4)),
(SynthSimulationForTests(simulation_config), hdf_storage_factory(), OneProcessPool()),
(SynthSimulationForTests(simulation_config), hdf_storage_factory(), ProcessPool(4)),
#(SynthSimulationForTests(simulation_config), hdf_storage_factory(), ThreadPool(4)),
(SynthSimulationWorkspace(simulation_config_workspace), Memory(), OneProcessPool(work_dir=work_dir)),
(SynthSimulationWorkspace(simulation_config_workspace), Memory(), ProcessPool(4, work_dir=work_dir)),
#(SynthSimulationWorkspace(simulation_config), storage_memory, ThreadPool(4, work_dir=work_dir)),
(SynthSimulationWorkspace(simulation_config_workspace), hdf_storage_factory(), OneProcessPool(work_dir=work_dir)),
(SynthSimulationWorkspace(simulation_config_workspace), hdf_storage_factory(), ProcessPool(4, work_dir=work_dir)),
#(SynthSimulationWorkspace(simulation_config), hdf_storage_factory(), ThreadPool(4, work_dir=work_dir))
]
)
def test_mlmc(test_case):
np.random.seed(1234)
def test_functions(self):
"""
Test numpy functions
"""
sample_storage = Memory()
result_format, sizes = self.fill_sample_storage(sample_storage)
root_quantity = make_root_quantity(sample_storage, result_format)
root_quantity_means = estimate_mean(root_quantity)
max_root_quantity = np.max(root_quantity, axis=0, keepdims=True)
max_means = estimate_mean(max_root_quantity)
assert len(max_means.mean) == 1
sin_root_quantity = np.sin(root_quantity)
sin_means = estimate_mean(sin_root_quantity)
assert len(sin_means.mean) == np.sum(sizes)
round_root_quantity = np.sum(root_quantity, axis=0, keepdims=True)
round_means = estimate_mean(round_root_quantity)
assert len(round_means.mean) == 1
add_root_quantity = np.add(
root_quantity, root_quantity) # Add arguments element-wise.
add_root_quantity_means = estimate_mean(add_root_quantity)
assert np.allclose(add_root_quantity_means.mean.flatten(),
(root_quantity_means.mean * 2))
x = np.ones(108)
add_one_root_quantity = np.add(
x, root_quantity) # Add arguments element-wise.
add_one_root_quantity_means = estimate_mean(add_one_root_quantity)
assert np.allclose(root_quantity_means.mean + np.ones((108, )),
add_one_root_quantity_means.mean.flatten())
x = np.ones(108)
divide_one_root_quantity = np.divide(
x, root_quantity) # Add arguments element-wise.
divide_one_root_quantity_means = estimate_mean(
divide_one_root_quantity)
assert np.all(divide_one_root_quantity_means.mean < 1)
# Test broadcasting
x = np.ones(108)
arctan2_one_root_quantity = np.arctan2(
x, root_quantity) # Add arguments element-wise.
arctan2_one_root_quantity_means = estimate_mean(
arctan2_one_root_quantity)
assert np.all(arctan2_one_root_quantity_means.mean < 1)
max_root_quantity = np.maximum(
root_quantity,
root_quantity) # Element-wise maximum of array elements.
max_root_quantity_means = estimate_mean(max_root_quantity)
assert np.allclose(max_root_quantity_means.mean.flatten(),
root_quantity_means.mean)
length = root_quantity['length']
sin_length = np.sin(length)
sin_means_length = estimate_mean(sin_length)
assert np.allclose(
(sin_means.mean[sizes[0]:sizes[0] + sizes[1]]).tolist(),
sin_means_length.mean.tolist())
q_and = np.logical_and(True, root_quantity)
self.assertRaises(TypeError, estimate_mean, q_and)
cache_clear()
x = np.ones((108, 5, 2))
self.assertRaises(ValueError, np.add, x, root_quantity)
x = np.ones((108, 5, 2))
self.assertRaises(ValueError, np.divide, x, root_quantity)
def test_condition(self):
"""
Test select method
"""
sample_storage = Memory()
result_format, size = self.fill_sample_storage(sample_storage)
root_quantity = make_root_quantity(sample_storage, result_format)
root_quantity_mean = estimate_mean(root_quantity)
all_root_quantity = root_quantity.select(
np.logical_or(0 < root_quantity, root_quantity < 10))
all_root_quantity_mean = estimate_mean(all_root_quantity)
assert np.allclose(root_quantity_mean.mean,
all_root_quantity_mean.mean)
selected_quantity = root_quantity.select(root_quantity < 0)
with self.assertRaises(Exception):
estimate_mean(selected_quantity)
all_root_quantity = root_quantity.select(0 < root_quantity)
all_root_quantity_mean = estimate_mean(all_root_quantity)
assert np.allclose(root_quantity_mean.mean,
all_root_quantity_mean.mean)
root_quantity_comp = root_quantity.select(
root_quantity == root_quantity)
root_quantity_comp_mean = estimate_mean(root_quantity_comp)
assert np.allclose(root_quantity_mean.mean,
root_quantity_comp_mean.mean)
root_quantity_comp = root_quantity.select(
root_quantity < root_quantity)
with self.assertRaises(Exception):
estimate_mean(root_quantity_comp)
#new_quantity = selected_quantity + root_quantity
#self.assertRaises(AssertionError, (selected_quantity + root_quantity))
# bound root quantity result - select the ones which meet conditions
mask = np.logical_and(0 < root_quantity, root_quantity < 10)
q_bounded = root_quantity.select(mask)
mean_q_bounded = estimate_mean(q_bounded)
q_bounded_2 = root_quantity.select(0 < root_quantity,
root_quantity < 10)
mean_q_bounded_2 = estimate_mean(q_bounded_2)
assert np.allclose(mean_q_bounded.mean, mean_q_bounded.mean)
quantity_add = root_quantity + root_quantity
q_add_bounded = quantity_add.select(0 < quantity_add,
quantity_add < 20)
means_add_bounded = estimate_mean(q_add_bounded)
assert np.allclose((means_add_bounded.mean), mean_q_bounded_2.mean * 2)
q_bounded = root_quantity.select(10 < root_quantity,
root_quantity < 20)
mean_q_bounded = estimate_mean(q_bounded)
q_add_bounded = quantity_add.select(20 < quantity_add,
quantity_add < 40)
means_add_bounded_2 = estimate_mean(q_add_bounded)
assert np.allclose((means_add_bounded_2.mean), mean_q_bounded.mean * 2)
q_add_bounded_3 = quantity_add.select(root_quantity < quantity_add)
means_add_bounded_3 = estimate_mean(q_add_bounded_3)
assert len(means_add_bounded_3.mean) == len(root_quantity_mean.mean)
q_add_bounded_4 = quantity_add.select(root_quantity > quantity_add)
with self.assertRaises(Exception):
estimate_mean(q_add_bounded_4)
q_add_bounded_5 = quantity_add.select(root_quantity < quantity_add,
root_quantity < 10)
means_add_bounded_5 = estimate_mean(q_add_bounded_5)
assert len(means_add_bounded_5.mean) == len(mean_q_bounded.mean)
length = root_quantity['length']
mean_length = estimate_mean(length)
quantity_lt = length.select(length < 10) # use just first sample
means_lt = estimate_mean(quantity_lt)
assert len(mean_length.mean) == len(means_lt.mean)
q_add_bounded_6 = quantity_add.select(root_quantity < quantity_add,
length < 1)
with self.assertRaises(Exception):
estimate_mean(q_add_bounded_6)
q_add_bounded_7 = quantity_add.select(root_quantity < quantity_add,
length < 10)
means_add_bounded_7 = estimate_mean(q_add_bounded_7)
assert np.allclose(means_add_bounded_7.mean, means_add_bounded.mean)
quantity_le = length.select(length <= 9) # use just first sample
means_le = estimate_mean(quantity_le)
assert len(mean_length.mean) == len(means_le.mean)
quantity_lt = length.select(length < 1) # no sample matches condition
with self.assertRaises(Exception):
estimate_mean(quantity_lt)
quantity_lt_gt = length.select(
9 < length, length < 20) # one sample matches condition
means_lt_gt = estimate_mean(quantity_lt_gt)
assert len(mean_length.mean) == len(means_lt_gt.mean)
quantity_gt = length.select(
10**5 < length) # no sample matches condition
with self.assertRaises(Exception):
estimate_mean(quantity_gt)
quantity_ge = length.select(
10**5 <= length) # no sample matches condition
with self.assertRaises(Exception):
estimate_mean(quantity_ge)
quantity_eq = length.select(1 == length)
with self.assertRaises(Exception):
estimate_mean(quantity_eq)
quantity_ne = length.select(-1 != length)
means_ne = estimate_mean(quantity_ne)
assert np.allclose((means_ne.mean).tolist(), mean_length.mean.tolist())
def multiproces_sampler_test():
np.random.seed(3)
n_moments = 5
failed_fraction = 0.1
distr = stats.norm(loc=1, scale=2)
step_range = [[0.01], [0.001], [0.0001]]
# Create simulation instance
simulation_config = dict(distr=distr,
complexity=2,
nan_fraction=failed_fraction,
sim_method='_sample_fn')
simulation_factory = SynthSimulation(simulation_config)
sample_storage = Memory()
sampling_pool = ProcessPool(4)
# Plan and compute samples
sampler = Sampler(sample_storage=sample_storage,
sampling_pool=sampling_pool,
sim_factory=simulation_factory,
level_parameters=step_range)
true_domain = distr.ppf([0.0001, 0.9999])
moments_fn = Legendre(n_moments, true_domain)
sampler.set_initial_n_samples()
#sampler.set_initial_n_samples([1000])
sampler.schedule_samples()
sampler.ask_sampling_pool_for_samples()
q_estimator = QuantityEstimate(sample_storage=sample_storage,
moments_fn=moments_fn,
sim_steps=step_range)
#
target_var = 1e-4
sleep = 0
add_coef = 0.1
# # @TODO: test
# # New estimation according to already finished samples
# variances, n_ops = q_estimator.estimate_diff_vars_regression(sampler._n_scheduled_samples)
# n_estimated = new_estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops,
# n_levels=sampler.n_levels)
#
# # Loop until number of estimated samples is greater than the number of scheduled samples
# while not sampler.process_adding_samples(n_estimated, sleep, add_coef):
# # New estimation according to already finished samples
# variances, n_ops = q_estimator.estimate_diff_vars_regression(sampler._n_scheduled_samples)
# n_estimated = new_estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops,
# n_levels=sampler.n_levels)
#
# print("n estimated ", n_estimated)
print("collected samples ", sampler._n_scheduled_samples)
means, vars = q_estimator.estimate_moments(moments_fn)
print("means ", means)
print("vars ", vars)
assert means[0] == 1
assert np.isclose(means[1], 0, atol=5 * 1e-2)
assert vars[0] == 0
sampler.schedule_samples()
sampler.ask_sampling_pool_for_samples()
storage = sampler.sample_storage
results = storage.sample_pairs()
def multiprocess_test():
np.random.seed(3)
n_moments = 5
distr = stats.norm(loc=1, scale=2)
step_range = [0.01, 0.001] #, 0.001, 0.0001]
os.chdir(os.path.dirname(os.path.realpath(__file__)))
work_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'_test_tmp')
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
shutil.copyfile('synth_sim_config.yaml',
os.path.join(work_dir, 'synth_sim_config.yaml'))
simulation_config = {
"config_yaml": os.path.join(work_dir, 'synth_sim_config.yaml')
}
simulation_factory = SynthSimulationWorkspace(simulation_config)
sample_storage = Memory()
sampling_pool = ProcessPool(4, work_dir=work_dir)
# Plan and compute samples
sampler = Sampler(sample_storage=sample_storage,
sampling_pool=sampling_pool,
sim_factory=simulation_factory,
step_range=step_range)
true_domain = distr.ppf([0.0001, 0.9999])
moments_fn = Legendre(n_moments, true_domain)
sampler.set_initial_n_samples()
#sampler.set_initial_n_samples([1000])
sampler.schedule_samples()
sampler.ask_sampling_pool_for_samples()
q_estimator = QuantityEstimate(sample_storage=sample_storage,
moments_fn=moments_fn,
sim_steps=step_range)
target_var = 1e-4
sleep = 0
add_coef = 0.1
# # @TODO: test
# # New estimation according to already finished samples
# variances, n_ops = q_estimator.estimate_diff_vars_regression(sampler._n_scheduled_samples)
# n_estimated = new_estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops,
# n_levels=sampler.n_levels)
# # Loop until number of estimated samples is greater than the number of scheduled samples
# while not sampler.process_adding_samples(n_estimated, sleep, add_coef):
# # New estimation according to already finished samples
# variances, n_ops = q_estimator.estimate_diff_vars_regression(sampler._n_scheduled_samples)
# n_estimated = new_estimator.estimate_n_samples_for_target_variance(target_var, variances, n_ops,
# n_levels=sampler.n_levels)
print("collected samples ", sampler._n_scheduled_samples)
means, vars = q_estimator.estimate_moments(moments_fn)
print("means ", means)
print("vars ", vars)
assert means[0] == 1
assert np.isclose(means[1], 0, atol=1e-2)
assert vars[0] == 0
sampler.schedule_samples()
sampler.ask_sampling_pool_for_samples()
storage = sampler.sample_storage
results = storage.sample_pairs()