def test_custom_event_with_arg_handlers_profiler(): true_event_handler_time = 0.1 true_max_epochs = 1 true_num_iters = 2 profiler = HandlersTimeProfiler() dummy_trainer = Engine(_do_nothing_update_fn) dummy_trainer.register_events("custom_event") profiler.attach(dummy_trainer) @dummy_trainer.on(Events.ITERATION_COMPLETED(every=1)) def trigger_custom_event(): dummy_trainer.fire_event("custom_event") args = [122, 324] @dummy_trainer.on("custom_event", args) def on_custom_event(args): time.sleep(true_event_handler_time) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) results = profiler.get_results() event_results = None for row in results: if row[1] == "custom_event": event_results = row break assert event_results is not None assert "on_custom_event" in event_results[0] assert event_results[2] == approx(true_max_epochs * true_num_iters * true_event_handler_time, abs=1e-1) # total assert event_results[3][0] == approx(true_event_handler_time, abs=1e-1) # min assert event_results[4][0] == approx(true_event_handler_time, abs=1e-1) # max assert event_results[5] == approx(true_event_handler_time, abs=1e-1) # mean assert event_results[6] == approx(0.0, abs=1e-1) # stddev
def test_processing_timer_handlers_profiler(): true_processing_time = 0.1 true_max_epochs = 2 true_num_iters = 2 def train_updater(engine, batch): time.sleep(true_processing_time) profiler = HandlersTimeProfiler() dummy_trainer = Engine(train_updater) profiler.attach(dummy_trainer) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) results = profiler.get_results() processing_results = results[-2] assert processing_results[0] == "Processing" # event name assert processing_results[1] == "None" # total assert processing_results[2] == approx(true_max_epochs * true_num_iters * true_processing_time, abs=1e-1) # min assert processing_results[3][0] == approx(true_processing_time, abs=1e-1) # max assert processing_results[4][0] == approx(true_processing_time, abs=1e-1) # mean assert processing_results[5] == approx(true_processing_time, abs=1e-1) # stddev assert processing_results[6] == approx(0.0, abs=1e-1)
def test_dataflow_timer_handlers_profiler(): true_dataflow_time_per_ele = 0.1 true_max_epochs = 1 true_num_iters = 2 def dummy_data_loader(data): while True: for d in data: time.sleep(true_dataflow_time_per_ele) yield d dummy_data = range(true_num_iters) profiler = HandlersTimeProfiler() dummy_trainer = Engine(_do_nothing_update_fn) profiler.attach(dummy_trainer) dummy_trainer.run(dummy_data_loader(dummy_data), max_epochs=true_max_epochs, epoch_length=true_num_iters) results = profiler.get_results() dataflow_results = results[-1] assert dataflow_results[0] == "Dataflow" # event name assert dataflow_results[1] == "None" # total assert dataflow_results[2] == approx(true_num_iters * true_dataflow_time_per_ele, abs=1e-1) # min assert dataflow_results[3][0] == approx(true_dataflow_time_per_ele, abs=1e-1) # max assert dataflow_results[4][0] == approx(true_dataflow_time_per_ele, abs=1e-1) # mean assert dataflow_results[5] == approx(true_dataflow_time_per_ele, abs=1e-1) # stddev assert dataflow_results[6] == approx(0.0, abs=1e-1)
def test_event_handler_get_batch_completed_handlers_profiler(): true_event_handler_time = 0.1 true_max_epochs = 1 true_num_iters = 2 profiler = HandlersTimeProfiler() dummy_trainer = Engine(_do_nothing_update_fn) profiler.attach(dummy_trainer) @dummy_trainer.on(Events.GET_BATCH_COMPLETED) def delay_get_batch_completed(engine): time.sleep(true_event_handler_time) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) results = profiler.get_results() event_results = results[0] assert "delay_get_batch_completed" in event_results[0] assert event_results[1] == "GET_BATCH_COMPLETED" assert event_results[2] == approx(true_max_epochs * true_num_iters * true_event_handler_time, abs=1e-1) # total assert event_results[3][0] == approx(true_event_handler_time, abs=1e-1) # min assert event_results[4][0] == approx(true_event_handler_time, abs=1e-1) # max assert event_results[5] == approx(true_event_handler_time, abs=1e-1) # mean assert event_results[6] == approx(0.0, abs=1e-1) # stddev
def test_event_handler_total_time_handlers_profiler(): true_event_handler_time = 0.125 true_max_epochs = 1 true_num_iters = 1 profiler = HandlersTimeProfiler() dummy_trainer, handlers_sleep_count, processing_sleep_count = get_prepared_engine_for_handlers_profiler( true_event_handler_time ) profiler.attach(dummy_trainer) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) results = profiler.get_results() total_handler_stats = results[-3] # total result row total_processing_stats = results[-2] # processing result row assert total_handler_stats[2] == approx(true_event_handler_time * handlers_sleep_count, abs=1e-1) # total time assert total_processing_stats[2] == approx(true_event_handler_time * processing_sleep_count, abs=1e-1) # total time
def test_neg_event_filter_threshold_handlers_profiler(): true_event_handler_time = 0.1 true_max_epochs = 1 true_num_iters = 1 profiler = HandlersTimeProfiler() dummy_trainer = Engine(_do_nothing_update_fn) profiler.attach(dummy_trainer) @dummy_trainer.on(Events.EPOCH_STARTED(once=2)) def do_something_once_on_2_epoch(): time.sleep(true_event_handler_time) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) results = profiler.get_results() event_results = results[0] assert "do_something_once_on_2_epoch" in event_results[0] assert event_results[1] == "EPOCH_STARTED" assert event_results[2] == "not triggered"
def test_event_handler_completed_handlers_profiler(): true_event_handler_time = 0.1 true_max_epochs = 2 true_num_iters = 2 profiler = HandlersTimeProfiler() dummy_trainer = Engine(_do_nothing_update_fn) profiler.attach(dummy_trainer) @dummy_trainer.on(Events.COMPLETED) def delay_complete(engine): time.sleep(true_event_handler_time) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) results = profiler.get_results() event_results = results[0] assert "delay_complete" in event_results[0] assert event_results[1] == "COMPLETED" assert event_results[2] == approx(true_event_handler_time, abs=1e-1) # total
def test_pos_event_filter_threshold_handlers_profiler(): true_event_handler_time = HandlersTimeProfiler.EVENT_FILTER_THESHOLD_TIME true_max_epochs = 2 true_num_iters = 1 profiler = HandlersTimeProfiler() dummy_trainer = Engine(_do_nothing_update_fn) profiler.attach(dummy_trainer) @dummy_trainer.on(Events.EPOCH_STARTED(once=2)) def do_something_once_on_2_epoch(): time.sleep(true_event_handler_time) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) results = profiler.get_results() event_results = results[0] assert "do_something_once_on_2_epoch" in event_results[0] assert event_results[1] == "EPOCH_STARTED" assert event_results[2] == approx( (true_max_epochs * true_num_iters * true_event_handler_time) / 2, abs=1e-1 ) # total
def test_profilers_wrong_inputs(): profiler = BasicTimeProfiler() with pytest.raises(TypeError, match=r"Argument engine should be ignite.engine.Engine"): profiler.attach(None) with pytest.raises(RuntimeError, match=r"Need pandas to write results as files"): with patch.dict("sys.modules", {"pandas": None}): profiler.write_results("") profiler = HandlersTimeProfiler() with pytest.raises(TypeError, match=r"Argument engine should be ignite.engine.Engine"): profiler.attach(None) with pytest.raises(RuntimeError, match=r"Need pandas to write results as files"): with patch.dict("sys.modules", {"pandas": None}): profiler.write_results("")
def test_print_results_handlers_profiler_handlers_profiler(capsys): true_max_epochs = 1 true_num_iters = 5 profiler = HandlersTimeProfiler() dummy_trainer, _, _ = get_prepared_engine_for_handlers_profiler(true_event_handler_time=0.0125) profiler.attach(dummy_trainer) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) HandlersTimeProfiler.print_results(profiler.get_results()) captured = capsys.readouterr() out = captured.out assert "HandlersTimeProfiler." not in out assert "Timer." not in out
def test_write_results_handlers_profiler(dirname): true_event_handler_time = 0.125 true_max_epochs = 3 true_num_iters = 2 profiler = HandlersTimeProfiler() dummy_trainer, _, _ = get_prepared_engine_for_handlers_profiler(true_event_handler_time) profiler.attach(dummy_trainer) dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs) fp = os.path.join(dirname, "test_log.csv") profiler.write_results(fp) assert os.path.isfile(fp) file_length = 0 with open(fp) as f: for _ in f: file_length += 1 assert file_length == (true_max_epochs * true_num_iters) + 1