def exercise_explore_completeness():
import libtbx.load_env
if (not libtbx.env.has_module("spotfinder")):
print("Skipping some tests due to missing module: spotfinder")
return
from libtbx.test_utils import contains_substring
from libtbx import easy_run
def run(args):
cmd = " ".join(["rstbx.simage.explore_completeness"] + args)
print(cmd)
buf = easy_run.fully_buffered(
command=cmd,
stdout_splitlines=False).raise_if_errors().stdout_buffer
for key in [
"Complete with ", "Observations per reflection:", " Median: "
]:
assert contains_substring(buf, key)
return buf
run(["d_min=10"])
args = ["d_min=10", "intensity_symmetry=P4", "use_symmetry=True"]
from libtbx import easy_mp
if (easy_mp.detect_problem() is None):
args.append("multiprocessing=True")
buf = run(args)
assert contains_substring(buf, 'lattice_symmetry = "P 4 2 2"')
def exercise_explore_completeness():
import libtbx.load_env
if (not libtbx.env.has_module("spotfinder")):
print "Skipping some tests due to missing module: spotfinder"
return
from libtbx.test_utils import contains_substring
from libtbx import easy_run
def run(args):
cmd = " ".join(["rstbx.simage.explore_completeness"] + args)
print cmd
buf = easy_run.fully_buffered(
command=cmd, stdout_splitlines=False).raise_if_errors().stdout_buffer
for key in [
"Complete with ",
"Observations per reflection:",
" Median: "]:
assert contains_substring(buf, key)
return buf
run(["d_min=10"])
args = ["d_min=10", "intensity_symmetry=P4", "use_symmetry=True"]
from libtbx import easy_mp
if (easy_mp.detect_problem() is None):
args.append("multiprocessing=True")
buf = run(args)
assert contains_substring(buf, 'lattice_symmetry = "P 4 2 2"')
def exercise_solver():
import libtbx.load_env
if (not libtbx.env.has_module("spotfinder")):
print "Skipping some tests due to missing module: spotfinder"
return
from libtbx.test_utils import block_show_diff, contains_substring
from libtbx import easy_run
def run(args):
cmd = " ".join(["rstbx.simage.solver"] + args)
print cmd
buf = easy_run.fully_buffered(
command=cmd, stdout_splitlines=False).raise_if_errors().stdout_buffer
for key in [
"Final:"]:
assert contains_substring(buf, key)
return buf
buf = run(["d_min=5"])
assert not block_show_diff(buf, """\
input_im0_i_perm: 0
Correlation of input and estimated I-obs:
i_perm=0: 1.00000 (r1: 0.00000)
""")
buf = run(["d_min=5", "lattice_symmetry=R32:R", "intensity_symmetry=R3:R"])
assert not block_show_diff(buf, """\
input_im0_i_perm: 1
Correlation of input and estimated I-obs:
i_perm=0: 0.06799 (r1: 0.37555)
i_perm=1: 1.00000 (r1: 0.00000)
""")
buf = run(["d_min=5", "lattice_symmetry=R32:R", "intensity_symmetry=P1"])
assert not block_show_diff(buf, """\
input_im0_i_perm: 5
Correlation of input and estimated I-obs:
i_perm=0: 0.07524 (r1: 0.38937)
i_perm=1: -0.02385 (r1: 0.40114)
i_perm=2: -0.04577 (r1: 0.41742)
i_perm=3: -0.00099 (r1: 0.40390)
i_perm=4: 0.00764 (r1: 0.39213)
i_perm=5: 1.00000 (r1: 0.00000)
""")
if (not libtbx.env.has_module("labelit")):
print "Skipping some tests due to missing module: labelit"
else:
from libtbx import easy_mp
mp_problem = easy_mp.detect_problem()
if (mp_problem is not None):
print "Skipping some tests:", mp_problem
else:
buf = run([
"d_min=5", "lattice_symmetry=P422", "intensity_symmetry=P4",
"index_and_integrate=True", "multiprocessing=True"])
assert contains_substring(buf, "Refined unit cell 9 (")
assert contains_substring(
buf, "Correlation of input and estimated I-obs:")
assert contains_substring(buf, " Best correlation: 0.999")
def exercise(exercise_fail):
exercise_func_wrapper_sub_directories()
from libtbx import easy_mp
mp_problem = easy_mp.detect_problem()
if (mp_problem is not None):
print "Skipping tst_easy_mp.py: %s" % mp_problem
return
check_if_stacktrace_is_propagated_properly(method='threading', nproc=2)
check_if_stacktrace_is_propagated_properly(method='multiprocessing',
nproc=2)
check_if_stacktrace_is_propagated_properly(method='threading', nproc=1)
check_if_stacktrace_is_propagated_properly(method='multiprocessing',
nproc=1)
data = potentially_large(size=1000)
eval_parallel(data)
assert len(easy_mp.fixed_func_registry) == 0
eval_parallel(data, func_wrapper=None, index_args=False)
assert len(easy_mp.fixed_func_registry) == 1
eval_parallel(data, func_wrapper=None, index_args=False, log=sys.stdout)
assert len(easy_mp.fixed_func_registry) == 2
sio = StringIO()
eval_parallel(data, func_wrapper=None, index_args=False, log=sio)
assert len(easy_mp.fixed_func_registry) == 3
lines = sio.getvalue().splitlines()
assert len(lines) == 2
assert lines[0].startswith("multiprocessing pool size: ")
assert lines[1].startswith("wall clock time: ")
eval_parallel(data, exercise_out_of_range=True)
if (exercise_fail):
eval_parallel(data, exercise_fail=True)
raise Exception_expected
results = easy_mp.pool_map(fixed_func=data,
args=range(1000),
processes=Auto)
del data
assert len(easy_mp.fixed_func_registry) == 0
#
from libtbx.clear_paths import \
remove_or_rename_files_and_directories_if_possible as clear
import os
op = os.path
def fixed_func(arg):
print "hello world", arg
return 10 * arg
def go():
return easy_mp.pool_map(fixed_func=fixed_func,
func_wrapper="sub_directories",
args=[1, 2])
clear(paths=["mp000", "mp001"])
results = go()
assert results == [(None, 10), (None, 20)]
for i in [1, 2]:
with open("mp%03d/log" % (i - 1), 'r') as fh:
assert fh.read().splitlines() == ["hello world %d" % i]
results = go()
assert results == [('sub-directory exists already: "mp000"', None),
('sub-directory exists already: "mp001"', None)]
clear(paths=["mp001"])
results = go()
assert results == [('sub-directory exists already: "mp000"', None),
(None, 20)]
clear(paths=["mp000", "mp001"])
results = easy_mp.pool_map(
fixed_func=fixed_func,
func_wrapper=easy_mp.func_wrapper_sub_directories(makedirs_mode=0),
args=[1, 2])
assert results == [('cannot chdir to sub-directory: "mp000"', None),
('cannot chdir to sub-directory: "mp001"', None)]
clear(paths=["mp000", "mp001"])
clear(paths=["bf000", "bf001"])
def bad_func(arg):
raise RuntimeError(str(arg))
results = easy_mp.pool_map(fixed_func=bad_func,
func_wrapper="sub_directories:bf",
args=[1, 2])
assert results == [('CAUGHT EXCEPTION: "bf000/log"', None),
('CAUGHT EXCEPTION: "bf001/log"', None)]
for i in [1, 2]:
assert open("bf%03d/log" % (i-1)).read().splitlines()[-1] \
== "RuntimeError: %d" % i
out = StringIO()
def simple_func(arg):
from math import log
x = float(arg)
y = 0
for i in range(2, 1000):
y += log(x) / log(float(i))
return y
def cb(result):
out.write("%.3f\n" % result)
result = easy_mp.pool_map(func=simple_func,
args=[1, 2, 3, 4, 5, 6, 7, 8],
call_back_for_serial_run=cb,
processes=1)
assert (out.getvalue() == """\
0.000
122.891
194.777
245.782
285.344
317.668
344.998
368.673
""")
def exercise_solver():
import libtbx.load_env
if (not libtbx.env.has_module("spotfinder")):
print("Skipping some tests due to missing module: spotfinder")
return
from libtbx.test_utils import block_show_diff, contains_substring
from libtbx import easy_run
def run(args):
cmd = " ".join(["rstbx.simage.solver"] + args)
print(cmd)
buf = easy_run.fully_buffered(
command=cmd,
stdout_splitlines=False).raise_if_errors().stdout_buffer
for key in ["Final:"]:
assert contains_substring(buf, key)
return buf
buf = run(["d_min=5"])
assert not block_show_diff(
buf, """\
input_im0_i_perm: 0
Correlation of input and estimated I-obs:
i_perm=0: 1.00000 (r1: 0.00000)
""")
buf = run(["d_min=5", "lattice_symmetry=R32:R", "intensity_symmetry=R3:R"])
assert not block_show_diff(
buf, """\
input_im0_i_perm: 1
Correlation of input and estimated I-obs:
i_perm=0: 0.06799 (r1: 0.37555)
i_perm=1: 1.00000 (r1: 0.00000)
""")
buf = run(["d_min=5", "lattice_symmetry=R32:R", "intensity_symmetry=P1"])
assert not block_show_diff(
buf, """\
input_im0_i_perm: 5
Correlation of input and estimated I-obs:
i_perm=0: 0.07524 (r1: 0.38937)
i_perm=1: -0.02385 (r1: 0.40114)
i_perm=2: -0.04577 (r1: 0.41742)
i_perm=3: -0.00099 (r1: 0.40390)
i_perm=4: 0.00764 (r1: 0.39213)
i_perm=5: 1.00000 (r1: 0.00000)
""")
if (not libtbx.env.has_module("labelit")):
print("Skipping some tests due to missing module: labelit")
else:
from libtbx import easy_mp
mp_problem = easy_mp.detect_problem()
if (mp_problem is not None):
print("Skipping some tests:", mp_problem)
else:
buf = run([
"d_min=5", "lattice_symmetry=P422", "intensity_symmetry=P4",
"index_and_integrate=True", "multiprocessing=True"
])
assert contains_substring(buf, "Refined unit cell 9 (")
assert contains_substring(
buf, "Correlation of input and estimated I-obs:")
assert contains_substring(buf, " Best correlation: 0.999")
def exercise(exercise_fail):
exercise_func_wrapper_sub_directories()
from libtbx import easy_mp
mp_problem = easy_mp.detect_problem()
if (mp_problem is not None):
print "Skipping tst_easy_mp.py: %s" % mp_problem
return
data = potentially_large(size=1000)
eval_parallel(data)
assert len(easy_mp.fixed_func_registry) == 0
eval_parallel(data, func_wrapper=None, index_args=False)
assert len(easy_mp.fixed_func_registry) == 1
eval_parallel(data, func_wrapper=None, index_args=False, log=sys.stdout)
assert len(easy_mp.fixed_func_registry) == 2
sio = StringIO()
eval_parallel(data, func_wrapper=None, index_args=False, log=sio)
assert len(easy_mp.fixed_func_registry) == 3
lines = sio.getvalue().splitlines()
assert len(lines) == 2
assert lines[0].startswith("multiprocessing pool size: ")
assert lines[1].startswith("wall clock time: ")
eval_parallel(data, exercise_out_of_range=True)
if (exercise_fail):
eval_parallel(data, exercise_fail=True)
raise Exception_expected
results = easy_mp.pool_map(fixed_func=data, args=range(1000), processes=Auto)
del data
assert len(easy_mp.fixed_func_registry) == 0
#
from libtbx.clear_paths import \
remove_or_rename_files_and_directories_if_possible as clear
from libtbx import only_element
import os
op = os.path
def fixed_func(arg):
print "hello world", arg
return 10*arg
def go():
return easy_mp.pool_map(
fixed_func=fixed_func,
func_wrapper="sub_directories",
args=[1,2])
clear(paths=["mp000", "mp001"])
results = go()
assert results == [(None, 10), (None, 20)]
for i in [1,2]:
only_element(open("mp%03d/log" % (i-1)).read().splitlines()) \
== "hello world %d" % i
results = go()
assert results == [
('sub-directory exists already: "mp000"', None),
('sub-directory exists already: "mp001"', None)]
clear(paths=["mp001"])
results = go()
assert results == [
('sub-directory exists already: "mp000"', None),
(None, 20)]
clear(paths=["mp000", "mp001"])
results = easy_mp.pool_map(
fixed_func=fixed_func,
func_wrapper=easy_mp.func_wrapper_sub_directories(makedirs_mode=0),
args=[1,2])
assert results == [
('cannot chdir to sub-directory: "mp000"', None),
('cannot chdir to sub-directory: "mp001"', None)]
clear(paths=["mp000", "mp001"])
clear(paths=["bf000", "bf001"])
def bad_func(arg):
raise RuntimeError(str(arg))
results = easy_mp.pool_map(
fixed_func=bad_func,
func_wrapper="sub_directories:bf",
args=[1,2])
assert results == [
('CAUGHT EXCEPTION: "bf000/log"', None),
('CAUGHT EXCEPTION: "bf001/log"', None)]
for i in [1,2]:
assert open("bf%03d/log" % (i-1)).read().splitlines()[-1] \
== "RuntimeError: %d" % i
out = StringIO()
def simple_func (arg) :
from math import log
x = float(arg)
y = 0
for i in range(2, 1000) :
y += log(x) / log(float(i))
return y
def cb (result) :
out.write("%.3f\n" % result)
result = easy_mp.pool_map(
func=simple_func,
args=[1,2,3,4,5,6,7,8],
call_back_for_serial_run=cb,
processes=1)
assert (out.getvalue() == """\
0.000
122.891
194.777
245.782
285.344
317.668
344.998
368.673
""")
