def test_custom_operator_profiling_naive_engine(): # run the three tests above using Naive Engine run_in_spawned_process(test_custom_operator_profiling, \ {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ 'test_custom_operator_profiling_naive.json') run_in_spawned_process(custom_operator_profiling_multiple_custom_ops, \ {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, 'imperative', \ 'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json') run_in_spawned_process(custom_operator_profiling_multiple_custom_ops, \ {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, 'symbolic', \ 'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json')
def test_custom_operator_profiling_naive_engine(): # run the three tests above using Naive Engine run_in_spawned_process(test_custom_operator_profiling, \ {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ 'test_custom_operator_profiling_naive.json') run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ {'MXNET_ENGINE_TYPE' : "NaiveEngine"}, \ 'imperative', \ 'test_custom_operator_profiling_multiple_custom_ops_imperative_naive.json') run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ {'MXNET_ENGINE_TYPE' : "NaiveEngine", \ 'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ 'symbolic', \ 'test_custom_operator_profiling_multiple_custom_ops_symbolic_naive.json')
def _test_bulking(test_bulking_func): # test case format: (max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training) test_cases = [(0, 0, True), (1, 1, True), (15, 15, False), (15, 0, True), (0, 15, True), (15, 15, True)] times = {} times_str = '' for seg_sizes in test_cases: # Create shared variable to return measured time from test process time_per_iteration = mp.Manager().Value('d', 0.0) if not run_in_spawned_process( test_bulking_func, { 'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD': str(seg_sizes[0]), 'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD': str(seg_sizes[1]), 'MXNET_EXEC_BULK_EXEC_TRAIN': str(seg_sizes[2]) }, time_per_iteration): # skip test since the python version can't run it properly. Warning msg was logged. return times[seg_sizes] = time_per_iteration.value times_str += \ '\n runtime of (fwd,bwd,enable) op seg setting ({},{},{}) =\t{:.1f} msec'.format( seg_sizes[0], seg_sizes[1], seg_sizes[2], 1000.0 * times[seg_sizes]) fastest_non_bulked_time = min(times[(0, 0, True)], times[(1, 1, True)], times[(15, 15, False)]) slowest_half_bulked_time = max(times[(0, 15, True)], times[(15, 0, True)]) fastest_half_bulked_time = min(times[(0, 15, True)], times[(15, 0, True)]) fully_bulked_time = times[(15, 15, True)] print(times_str) # Non-bulked times[0,0,True], times[1,1,True] and times[15,15,False] should be about the same, # slower than both half-bulked times[0,15,True] and times[15,0,True] assert slowest_half_bulked_time < fastest_non_bulked_time, \ 'A half-bulked exec time is slower than the non-bulked time by {} secs! {}' \ .format(slowest_half_bulked_time - fastest_non_bulked_time, times_str) # The fully bulked times[15,15,True] should be faster than both half-bulked runs assert fully_bulked_time < fastest_half_bulked_time, \ 'The fully-bulked exec time is slower than a half-bulked time by {} secs! {}' \ .format(fully_bulked_time - fastest_half_bulked_time, times_str)
def test_bulking(): # test case format: (max_fwd_segment_size, max_bwd_segment_size, enable_bulking_in_training) test_cases = [(0, 0, True), (1, 1, True), (15, 15, False), (15, 0, True), (0, 15, True), (15, 15, True)] times = {} times_str = '' for seg_sizes in test_cases: # Create shared variable to return measured time from test process time_per_iteration = mp.Manager().Value('d', 0.0) if not run_in_spawned_process(_test_bulking_in_process, {'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD': seg_sizes[0], 'MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD': seg_sizes[1], 'MXNET_EXEC_BULK_EXEC_TRAIN': seg_sizes[2]}, time_per_iteration): # skip test since the python version can't run it properly. Warning msg was logged. return times[seg_sizes] = time_per_iteration.value times_str += \ '\n runtime of (fwd,bwd,enable) op seg setting ({},{},{}) =\t{:.1f} msec'.format( seg_sizes[0], seg_sizes[1], seg_sizes[2], 1000.0 * times[seg_sizes]) fastest_non_bulked_time = min( times[(0, 0, True)], times[(1, 1, True)], times[(15, 15, False)]) slowest_half_bulked_time = max(times[(0, 15, True)], times[(15, 0, True)]) fastest_half_bulked_time = min(times[(0, 15, True)], times[(15, 0, True)]) fully_bulked_time = times[(15, 15, True)] print(times_str) # Non-bulked times[0,0,True], times[1,1,True] and times[15,15,False] should be about the same, # slower than both half-bulked times[0,15,True] and times[15,0,True] assert slowest_half_bulked_time < fastest_non_bulked_time, \ 'A half-bulked exec time is slower than the non-bulked time by {} secs! {}' \ .format(slowest_half_bulked_time - fastest_non_bulked_time, times_str) # The fully bulked times[15,15,True] should be faster than both half-bulked runs assert fully_bulked_time < fastest_half_bulked_time, \ 'The fully-bulked exec time is slower than a half-bulked time by {} secs! {}' \ .format(fully_bulked_time - fastest_half_bulked_time, times_str)
def test_custom_operator_profiling_multiple_custom_ops_symbolic(): run_in_spawned_process(test_custom_operator_profiling_multiple_custom_ops_imperative, \ {'MXNET_EXEC_BULK_EXEC_INFERENCE' : 0, \ 'MXNET_EXEC_BULK_EXEC_TRAIN' : 0}, \ 'symbolic', \ 'test_custom_operator_profiling_multiple_custom_ops_symbolic.json')