def test_run_parallel_jobs():
"""Tests morpheus_core.helpers.parallel_helper.run_parallel_jobs"""
helper.setup()
workers = ["0", "1"]
for w in workers:
os.mkdir(os.path.join(helper.TMP_DIR, w))
with open(os.path.join(helper.TMP_DIR, w, "main.py"), "w") as f:
f.write("\n".join([
"import time",
"def main():",
" time.sleep(0.15)",
"",
"if __name__=='__main__':",
" main()",
]))
is_gpu = False
out_dir = helper.TMP_DIR
parallel_check_interval = 0.2
start_time = time.time()
ph.run_parallel_jobs(workers, is_gpu, out_dir, parallel_check_interval)
end_time = time.time()
run_time = end_time - start_time
expected_max_time = 0.6
# if the process runs in parallel then they will both be done before the
# first check interval
assert run_time < expected_max_time
helper.tear_down()
def test_open_file():
"""Tests morpheus_core.helpers.fits_helper.open_file"""
helper.setup()
sample_location = helper.make_sample_file()
expected_array = np.arange(100).reshape([10, 10])
hdul, actual_array = fh.open_file(sample_location)
np.testing.assert_array_equal(expected_array, actual_array)
helper.tear_down()
def test_create_file():
"""Tests morpheus_core.helpers.fits_helper.create_file"""
helper.setup()
shape = (100, 100)
tmp_out = os.path.join(helper.TMP_DIR, "test.fits")
fh.create_file(tmp_out, shape, np.float32)
actual = fits.getdata(tmp_out)
assert actual.shape == shape
helper.tear_down()
def test_stitch_parallel_classifications():
"""tests morpheus_core.helpers.parallel_helper.stitch_parallel_classifications"""
helper.setup()
out_dir = helper.TMP_DIR
test_shape = [10, 10, 1, 2]
window_shape = [5, 5]
workers = [0, 1]
aggregation_method = morpheus_core.AGGREGATION_METHODS.MEAN_VAR
for w in workers:
os.mkdir(os.path.join(out_dir, str(w)))
os.mkdir(os.path.join(out_dir, str(w), "output"))
test_data = np.zeros(test_shape)
test_data[:, :, 0, 0] = 1 + w
fits.PrimaryHDU(data=test_data).writeto(
os.path.join(TMP_DIR, str(w), "output", "output.fits"))
fits.PrimaryHDU(data=np.ones(test_shape[:2])).writeto(
os.path.join(TMP_DIR, str(w), "output", "n.fits"))
hduls, outputs = ph.stitch_parallel_classifications(
workers, out_dir, aggregation_method, window_shape)
actual_output, actual_n = outputs
expected_n = np.ones([16, 10])
expected_n[6:10, :] = 2
expected_output = np.zeros([16, 10, 1, 2])
expected_output[:6, :, 0, 0] = 1
expected_output[6:10, :, 0, 0] = 1.5
expected_output[10:, :, 0, 0] = 2
expected_output[:6, :, 0, 1] = 0
expected_output[6:10, :, 0, 1] = 0.25
expected_output[10:, :, 0, 1] = 0
np.testing.assert_array_equal(actual_n, expected_n)
np.testing.assert_array_equal(actual_output, expected_output)
list(map(lambda h: h.close(), hduls))
helper.tear_down()
def test_get_rank_vote_array_on_disk():
"""tests morpheus_core.helpers.label_helper.get_mean_var_array"""
helper.setup()
shape = [100, 100, 1]
out_file = os.path.join(helper.TMP_DIR, "test.fits")
hdul, in_mem_array = lh.get_rank_vote_array(shape, write_to=out_file)
on_disk_array = fits.getdata(out_file)
# validate the returned array
assert hdul is not None
assert in_mem_array.shape == (100, 100, 1)
assert on_disk_array.shape == (100, 100, 1)
helper.tear_down()
def test_predict_mean_var_on_disk():
"""Tests morpheus_core.predict with mean_var aggergation on disk"""
helpers.setup()
model = lambda x: np.ones_like(x[0])
model_inputs = [helpers.make_mock_input()]
n_classes = 1
batch_size = 10
window_size = (10, 10)
stride = (1, 1)
update_map = np.ones(window_size)
aggregate_method = morpheus_core.AGGREGATION_METHODS.MEAN_VAR
out_dir = helpers.TMP_DIR
gpus = None
cpus = None
parallel_check_interval = 1
hduls, outputs = morpheus_core.predict(
model,
model_inputs,
n_classes,
batch_size,
window_size,
stride,
update_map,
aggregate_method,
out_dir,
gpus,
cpus,
parallel_check_interval,
)
out_lbl, out_n = outputs
disk_lbl = fits.getdata(os.path.join(helpers.TMP_DIR, "output.fits"))
disk_n = fits.getdata(os.path.join(helpers.TMP_DIR, "n.fits"))
assert out_lbl.sum() == 100 * 100
assert out_lbl.shape == (100, 100, 1, 2)
np.testing.assert_array_equal(disk_lbl, out_lbl)
np.testing.assert_array_equal(disk_n, out_n)
np.testing.assert_array_equal(out_n[0, :10], np.arange(10) + 1)
misc.apply(lambda x: x.close(), hduls)
helpers.tear_down()
def test_get_empty_output_array_mean_var():
"""Tests morpheus_core.helpers.parallel_helper.get_empty_output_array for mean_var"""
helper.setup()
out_dir = helper.TMP_DIR
method = morpheus_core.AGGREGATION_METHODS.MEAN_VAR
in_shape = (100, 100, 5)
expected_output_shape = (100, 100, 5, 2)
expected_n_shape = (100, 100)
out_hdul, n_hdul, actual_output, actual_n = ph.get_empty_output_array(
out_dir, *in_shape, method)
assert actual_output.shape == expected_output_shape
assert actual_n.shape == expected_n_shape
out_hdul.close()
n_hdul.close()
helper.tear_down()
def test_build_parallel_classification_structure():
"""Tests morpheus_core.helpers.parallel_helper.build_parallel_classification_structure"""
helper.setup()
model = pickle_rick
arr_fnames = [helper.make_mock_input()]
arrs = [fits.getdata(arr_fnames[0])]
n_classes = 2
batch_size = 10
window_size = (10, 10)
stride = (1, 1)
update_map = np.ones(window_size)
aggregate_method = morpheus_core.AGGREGATION_METHODS.MEAN_VAR
out_dir = helper.TMP_DIR
workers = [0, 1]
ph.build_parallel_classification_structure(
model,
arrs,
arr_fnames,
n_classes,
batch_size,
window_size,
stride,
update_map,
aggregate_method,
out_dir,
workers,
)
expected_parent_tree = set(["input.fits", "0", "1"])
expected_sub_tree = set(
["input.fits", "main.py", "model.pkl", "update_map.npy"])
assert expected_parent_tree == set(os.listdir(out_dir))
assert expected_sub_tree == set(os.listdir(os.path.join(out_dir, "0")))
assert expected_sub_tree == set(os.listdir(os.path.join(out_dir, "1")))
helper.tear_down()
def test_get_data_from_worker():
"""Tests morpheus_core.helpers.parallel_helper.get_data_from_worker"""
helper.setup()
out_dir = helper.TMP_DIR
worker = "1"
os.mkdir(os.path.join(out_dir, worker))
os.mkdir(os.path.join(out_dir, worker, "output"))
output_path = os.path.join(out_dir, worker, "output", "output.fits")
n_path = os.path.join(out_dir, worker, "output", "n.fits")
mock_data = np.zeros([100, 100], dtype=np.float32)
fits.PrimaryHDU(data=mock_data).writeto(output_path)
fits.PrimaryHDU(data=mock_data).writeto(n_path)
actual_output, actual_n = ph.get_data_from_worker(out_dir, worker)
np.testing.assert_array_equal(mock_data, actual_output)
np.testing.assert_array_equal(mock_data, actual_n)
helper.tear_down()
def test_make_runnable_file():
"""Tests morpheus_core.helpers.parallel_helper.make_runnable_file"""
helper.setup()
path = helper.TMP_DIR
input_file_names = [helper.make_sample_file()]
n_classes = 2
batch_size = 10
window_size = (10, 10)
stride = (1, 1)
aggregate_method = morpheus_core.AGGREGATION_METHODS.MEAN_VAR
local = os.path.dirname(os.path.dirname(__file__))
expected_text = [
"import sys",
f"sys.path.append('{local}')",
"import os",
"import dill",
"import numpy as np",
"from tqdm import tqdm",
"from morpheus_core import morpheus_core",
"def main():",
" output_dir = './output'",
" if 'output' not in os.listdir():",
" os.mkdir('./output')",
"",
" with open('model.pkl', 'rb') as f:",
" model = dill.load(f)",
"",
" model_inputs = [",
" " + ",".join(["'" + i + "'" for i in input_file_names]),
" ]",
"",
" update_map = np.load('update_map.npy', allow_pickle=True)",
"",
" morpheus_core.predict(",
" model,",
" model_inputs,",
f" {n_classes},",
f" {batch_size},",
f" {window_size},",
f" stride={stride},",
" update_map=update_map,",
f" aggregate_method='{aggregate_method}',",
" out_dir=output_dir,",
" )",
" sys.exit(0)",
"if __name__=='__main__':",
" main()",
]
ph.make_runnable_file(
path,
input_file_names,
n_classes,
batch_size,
window_size,
stride,
aggregate_method,
)
with open(os.path.join(path, "main.py"), "r") as f:
actual_text = [l.rstrip() for l in f.readlines()]
assert expected_text == actual_text
helper.tear_down()