def test_custom_operator_profiling_naive_engine():
# run the three tests above using Naive Engine
run_in_spawned_process(test_custom_operator_profiling, \
{'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \
'test_custom_operator_profiling_naive.json')
run_in_spawned_process(custom_operator_profiling_multiple_custom_ops, \
{'MXNET_ENGINE_TYPE' : "NaiveEngine"}, 'imperative', \
'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json')
run_in_spawned_process(custom_operator_profiling_multiple_custom_ops, \
{'MXNET_ENGINE_TYPE' : "NaiveEngine"}, 'symbolic', \
'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json')
def test_custom_operator_profiling_naive_engine():
# run the three tests above using Naive Engine
run_in_spawned_process(test_custom_operator_profiling, \
{'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \
'test_custom_operator_profiling_naive.json')
run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \
{'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \
'imperative', \
'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json')
run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \
{'MXNET_ENGINE_TYPE' : "NaiveEngine", \
'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \
'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \
'symbolic', \
'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json')
def _test_bulking(test_bulking_func):
# test case format: (max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training)
test_cases = [(0, 0, True), (1, 1, True), (15, 15, False), (15, 0, True),
(0, 15, True), (15, 15, True)]
times = {}
times_str = ''
for seg_sizes in test_cases:
# Create shared variable to return measured time from test process
time_per_iteration = mp.Manager().Value('d', 0.0)
if not run_in_spawned_process(
test_bulking_func,
{
'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD': str(seg_sizes[0]),
'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD': str(seg_sizes[1]),
'MXNET_EXEC_BULK_EXEC_TRAIN': str(seg_sizes[2])
}, time_per_iteration):
# skip test since the python version can't run it properly. Warning msg was logged.
return
times[seg_sizes] = time_per_iteration.value
times_str += \
'\n runtime of (fwd,bwd,enable) op seg setting ({},{},{}) =\t{:.1f} msec'.format(
seg_sizes[0], seg_sizes[1], seg_sizes[2], 1000.0 * times[seg_sizes])
fastest_non_bulked_time = min(times[(0, 0, True)], times[(1, 1, True)],
times[(15, 15, False)])
slowest_half_bulked_time = max(times[(0, 15, True)], times[(15, 0, True)])
fastest_half_bulked_time = min(times[(0, 15, True)], times[(15, 0, True)])
fully_bulked_time = times[(15, 15, True)]
print(times_str)
# Non-bulked times[0,0,True], times[1,1,True] and times[15,15,False] should be about the same,
# slower than both half-bulked times[0,15,True] and times[15,0,True]
assert slowest_half_bulked_time < fastest_non_bulked_time, \
'A half-bulked exec time is slower than the non-bulked time by {} secs! {}' \
.format(slowest_half_bulked_time - fastest_non_bulked_time, times_str)
# The fully bulked times[15,15,True] should be faster than both half-bulked runs
assert fully_bulked_time < fastest_half_bulked_time, \
'The fully-bulked exec time is slower than a half-bulked time by {} secs! {}' \
.format(fully_bulked_time - fastest_half_bulked_time, times_str)
def test_bulking():
# test case format: (max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training)
test_cases = [(0, 0, True), (1, 1, True), (15, 15, False),
(15, 0, True), (0, 15, True), (15, 15, True)]
times = {}
times_str = ''
for seg_sizes in test_cases:
# Create shared variable to return measured time from test process
time_per_iteration = mp.Manager().Value('d', 0.0)
if not run_in_spawned_process(_test_bulking_in_process,
{'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD': seg_sizes[0],
'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD': seg_sizes[1],
'MXNET_EXEC_BULK_EXEC_TRAIN': seg_sizes[2]},
time_per_iteration):
# skip test since the python version can't run it properly. Warning msg was logged.
return
times[seg_sizes] = time_per_iteration.value
times_str += \
'\n runtime of (fwd,bwd,enable) op seg setting ({},{},{}) =\t{:.1f} msec'.format(
seg_sizes[0], seg_sizes[1], seg_sizes[2], 1000.0 * times[seg_sizes])
fastest_non_bulked_time = min(
times[(0, 0, True)], times[(1, 1, True)], times[(15, 15, False)])
slowest_half_bulked_time = max(times[(0, 15, True)], times[(15, 0, True)])
fastest_half_bulked_time = min(times[(0, 15, True)], times[(15, 0, True)])
fully_bulked_time = times[(15, 15, True)]
print(times_str)
# Non-bulked times[0,0,True], times[1,1,True] and times[15,15,False] should be about the same,
# slower than both half-bulked times[0,15,True] and times[15,0,True]
assert slowest_half_bulked_time < fastest_non_bulked_time, \
'A half-bulked exec time is slower than the non-bulked time by {} secs! {}' \
.format(slowest_half_bulked_time - fastest_non_bulked_time, times_str)
# The fully bulked times[15,15,True] should be faster than both half-bulked runs
assert fully_bulked_time < fastest_half_bulked_time, \
'The fully-bulked exec time is slower than a half-bulked time by {} secs! {}' \
.format(fully_bulked_time - fastest_half_bulked_time, times_str)
def test_custom_operator_profiling_multiple_custom_ops_symbolic():
run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \
{'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \
'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \
'symbolic', \
'test_custom_operator_profiling_multiple_custom_ops_symbolic.json')
