def test_concat_scheduler_asserts(): tensor = torch.zeros([1], requires_grad=True) optimizer = torch.optim.SGD([tensor], lr=0) scheduler_1 = LinearCyclicalScheduler(optimizer, "lr", start_value=1.0, end_value=0.0, cycle_size=10) scheduler_2 = CosineAnnealingScheduler(optimizer, "lr", start_value=0.0, end_value=1.0, cycle_size=10) with pytest.raises(ValueError): ConcatScheduler(schedulers=[], durations=[]) with pytest.raises(ValueError): ConcatScheduler(schedulers=[scheduler_1], durations=[10]) with pytest.raises(TypeError): ConcatScheduler(schedulers=[scheduler_1, 12], durations=[10]) with pytest.raises(ValueError): ConcatScheduler(schedulers=[scheduler_1, scheduler_2], durations=[10, 5]) with pytest.raises(ValueError): ConcatScheduler(schedulers=[scheduler_1, scheduler_2, scheduler_2], durations=[15, 12.0]) with pytest.raises(ValueError): ConcatScheduler(schedulers=[scheduler_1, scheduler_2], durations="abc") with pytest.raises(ValueError): ConcatScheduler.simulate_values( num_events=123, schedulers=[scheduler_1, scheduler_2], durations=[15], param_names="abc" )
def test_concat_scheduler_asserts(): tensor = torch.zeros([1], requires_grad=True) optimizer = torch.optim.SGD([tensor], lr=0) scheduler_1 = LinearCyclicalScheduler(optimizer, "lr", start_value=1.0, end_value=0.0, cycle_size=10) scheduler_2 = CosineAnnealingScheduler(optimizer, "lr", start_value=0.0, end_value=1.0, cycle_size=10) with pytest.raises(TypeError, match=r"Argument schedulers should be a sequence"): ConcatScheduler(schedulers=None, durations=[]) with pytest.raises(ValueError, match=r"Argument schedulers should be of more than one parameter schedulers"): ConcatScheduler(schedulers=[], durations=[]) with pytest.raises(ValueError, match=r"Argument schedulers should be of more than one parameter schedulers"): ConcatScheduler(schedulers=[scheduler_1], durations=[10]) with pytest.raises(TypeError, match=r"Value at index 1 of schedulers should be a parameter scheduler"): ConcatScheduler(schedulers=[scheduler_1, 12], durations=[10]) with pytest.raises(ValueError, match=r"Incorrect number schedulers or duration values"): ConcatScheduler(schedulers=[scheduler_1, scheduler_2], durations=[10, 5]) with pytest.raises(ValueError, match=r"Argument durations should be list/tuple of integers"): ConcatScheduler(schedulers=[scheduler_1, scheduler_2, scheduler_2], durations=[15, 12.0]) with pytest.raises(TypeError, match=r"Argument durations should be list/tuple"): ConcatScheduler(schedulers=[scheduler_1, scheduler_2], durations="abc") with pytest.raises(TypeError, match=r"Argument param_names should be list or tuple"): ConcatScheduler.simulate_values( num_events=123, schedulers=[scheduler_1, scheduler_2], durations=[15], param_names="abc" ) with pytest.raises(ValueError, match=r"Argument param_names should be list or tuple of strings"): ConcatScheduler.simulate_values( num_events=123, schedulers=[scheduler_1, scheduler_2], durations=[15], param_names=[1] ) optimizer_2 = torch.optim.SGD([tensor], lr=0) scheduler_3 = CosineAnnealingScheduler(optimizer_2, "lr", start_value=0.0, end_value=1.0, cycle_size=10) with pytest.raises(ValueError, match=r"schedulers should be related to same optimizer"): ConcatScheduler([scheduler_1, scheduler_3], durations=[30,])
def test_concat_scheduler_two_linear(): tensor = torch.zeros([1], requires_grad=True) optimizer = torch.optim.SGD([tensor], lr=0) scheduler_1 = LinearCyclicalScheduler(optimizer, "lr", start_value=0.0, end_value=0.1, cycle_size=2) scheduler_2 = LinearCyclicalScheduler(optimizer, "lr", start_value=0.2, end_value=1.0, cycle_size=2) durations = [5, ] concat_scheduler = ConcatScheduler(schedulers=[scheduler_1, scheduler_2], durations=durations, save_history=True) assert concat_scheduler.get_param() == 0.0 data = [0] * 10 max_epochs = 2 simulated_values = ConcatScheduler.simulate_values(num_events=len(data) * max_epochs, schedulers=[scheduler_1, scheduler_2], durations=durations) lrs = [] def save_lr(engine): lrs.append(optimizer.param_groups[0]['lr']) trainer = Engine(lambda engine, batch: None) trainer.add_event_handler(Events.ITERATION_STARTED, concat_scheduler) trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr) trainer.run(data, max_epochs=max_epochs) assert lrs == list(map(pytest.approx, [ # first LinearCyclicalScheduler 0.0, 0.1, 0.0, 0.1, 0.0, # second LinearCyclicalScheduler 0.2, 1.0, 0.2, 1.0, 0.2, 1.0, 0.2, 1.0, 0.2, 1.0, 0.2, 1.0, 0.2, 1.0, 0.2, ])) state_lrs = trainer.state.param_history['lr'] assert len(state_lrs) == len(lrs) # Unpack singleton lists assert [group[0] for group in state_lrs] == lrs assert lrs == pytest.approx([v for i, v in simulated_values])
def test_concat_scheduler_3_schedulers(): tensor = torch.zeros([1], requires_grad=True) optimizer = torch.optim.SGD([tensor], lr=0) scheduler_1 = LinearCyclicalScheduler(optimizer, "lr", start_value=1.0, end_value=0.5, cycle_size=20) scheduler_2 = LinearCyclicalScheduler(optimizer, "lr", start_value=0.5, end_value=0.45, cycle_size=10) scheduler_3 = LinearCyclicalScheduler(optimizer, "lr", start_value=0.5, end_value=0.0, cycle_size=20) durations = [10, 5] concat_scheduler = ConcatScheduler(schedulers=[scheduler_1, scheduler_2, scheduler_3], durations=durations, save_history=True) data = [0] * 10 max_epochs = 2 simulated_values = ConcatScheduler.simulate_values(num_events=len(data) * max_epochs, schedulers=[scheduler_1, scheduler_2, scheduler_3], durations=durations) lrs = [] def save_lr(engine): lrs.append(optimizer.param_groups[0]['lr']) trainer = Engine(lambda engine, batch: None) trainer.add_event_handler(Events.ITERATION_STARTED, concat_scheduler) trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr) trainer.run(data, max_epochs=max_epochs) assert lrs == list(map(pytest.approx, [ # Cycle 1 of the first LinearCyclicalScheduler 1.0, 0.95, 0.9, 0.85, 0.8, 0.75, 0.7, 0.65, 0.6, 0.55, # Cycle 1 of the second LinearCyclicalScheduler 0.5, 0.49, 0.48, 0.47, 0.46, # Cycle 1 of the third LinearCyclicalScheduler 0.5, 0.45, 0.4, 0.35, 0.3, ])) state_lrs = trainer.state.param_history['lr'] assert len(state_lrs) == len(lrs) # Unpack singleton lists assert [group[0] for group in state_lrs] == lrs assert lrs == pytest.approx([v for i, v in simulated_values])
def _test(duration_vals_as_np_int): scheduler_1 = LinearCyclicalScheduler(optimizer, "lr", start_value=1.0, end_value=0.0, cycle_size=10) scheduler_2 = CosineAnnealingScheduler(optimizer, "lr", start_value=0.0, end_value=1.0, cycle_size=10) durations = [10, ] if duration_vals_as_np_int: durations = [np.int64(t) for t in durations] concat_scheduler = ConcatScheduler(schedulers=[scheduler_1, scheduler_2], durations=durations, save_history=True) data = [0] * 10 max_epochs = 2 simulated_values = ConcatScheduler.simulate_values(num_events=len(data) * max_epochs, schedulers=[scheduler_1, scheduler_2], durations=durations) lrs = [] def save_lr(engine): lrs.append(optimizer.param_groups[0]['lr']) trainer = Engine(lambda engine, batch: None) trainer.add_event_handler(Events.ITERATION_STARTED, concat_scheduler) trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr) trainer.run(data, max_epochs=max_epochs) assert lrs == list(map(pytest.approx, [ # Cycle 1 of the LinearCyclicalScheduler 1.0, 0.8, 0.6, 0.4, 0.2, 0.0, 0.2, 0.4, 0.6, 0.8, # Cycle 1 of the CosineAnnealingScheduler 0.0, 0.02447174185242318, 0.09549150281252627, 0.20610737385376332, 0.3454915028125263, 0.5, 0.6545084971874737, 0.7938926261462365, 0.9045084971874737, 0.9755282581475768, ])) state_lrs = trainer.state.param_history['lr'] assert len(state_lrs) == len(lrs) # Unpack singleton lists assert [group[0] for group in state_lrs] == lrs assert lrs == pytest.approx([v for i, v in simulated_values])