def test_param_group_scheduler_asserts():
t1 = torch.zeros([1], requires_grad=True)
t2 = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([{"params": t1, "lr": 0.1}, {"params": t2, "lr": 0.1}])
lr_scheduler1 = LinearCyclicalScheduler(
optimizer, "lr", param_group_index=0, start_value=1.0, end_value=0.0, cycle_size=10
)
lr_scheduler2 = LinearCyclicalScheduler(
optimizer, "lr", param_group_index=1, start_value=1.0, end_value=0.0, cycle_size=10
)
with pytest.raises(TypeError, match=r"Argument schedulers should be a list/tuple"):
ParamGroupScheduler(schedulers=None, names=["a", "b", "c"])
with pytest.raises(ValueError, match=r"Argument schedulers should be a list/tuple of parameter schedulers"):
ParamGroupScheduler(schedulers=[0, 1, 2], names=["a", "b", "c"])
with pytest.raises(ValueError, match=r"Argument schedulers should be a list/tuple of parameter schedulers"):
ParamGroupScheduler(schedulers=[lr_scheduler1, "2"], names=["a", "b"])
with pytest.raises(TypeError, match=r"Argument names should be a list/tuple"):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2], names="ab")
with pytest.raises(ValueError, match=r"Argument names should be a list/tuple of parameter scheduler's names"):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2], names=[1, 2])
with pytest.raises(ValueError, match=r"\d should be equal \d"):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2], names=["a"])
scheduler = ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2], names=["a", "b"])
with pytest.raises(ValueError, match=r"Required state attribute 'schedulers' is absent in provided state_dict"):
scheduler.load_state_dict({"a": 1})
with pytest.raises(ValueError, match=r"Input state_dict contains 0 state_dicts of param group schedulers"):
scheduler.load_state_dict({"schedulers": []})
with pytest.raises(ValueError, match=r"Required state attribute 'schedulers' is absent in provided state_dict"):
scheduler.load_state_dict({})
with pytest.raises(
ValueError, match=r"Name of scheduler from input state dict does not " r"correspond to required one"
):
scheduler.load_state_dict({"schedulers": [("a", lr_scheduler1.state_dict()), ("bad_name", {})]})
optimizer2 = torch.optim.SGD([{"params": t1, "lr": 0.1}, {"params": t2, "lr": 0.1}])
lr_scheduler3 = LinearCyclicalScheduler(
optimizer2, "lr", param_group_index=0, start_value=1.0, end_value=0.0, cycle_size=10
)
with pytest.raises(ValueError, match=r"schedulers should be related to same optimizer"):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler3])
def test_linear_scheduler():
with pytest.raises(
TypeError,
match=r"Argument optimizer should be torch.optim.Optimizer"):
LinearCyclicalScheduler({}, "lr", 1, 0, cycle_size=0)
tensor = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0.0)
with pytest.raises(
ValueError,
match=r"Argument cycle_size should be positive and larger than 1"):
LinearCyclicalScheduler(optimizer, "lr", 1, 0, cycle_size=0)
with pytest.raises(
ValueError,
match=r"Argument cycle_size should be positive and larger than 1"):
LinearCyclicalScheduler(optimizer, "lr", 1, 0, cycle_size=1)
scheduler = LinearCyclicalScheduler(optimizer, "lr", 1, 0, 10)
state_dict = scheduler.state_dict()
def save_lr(engine):
lrs.append(optimizer.param_groups[0]["lr"])
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run([0] * 9, max_epochs=2)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
# Cycle 2
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4, # 0.6, 0.8,
],
))
scheduler.load_state_dict(state_dict)
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler = LinearCyclicalScheduler(optimizer,
"lr",
1,
0,
10,
cycle_mult=2)
state_dict = scheduler.state_dict()
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run([0] * 10, max_epochs=3)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
# Cycle 2
1.0,
0.9,
0.8,
0.7,
0.6,
0.5,
0.4,
0.3,
0.2,
0.1,
0.0,
0.1,
0.2,
0.3,
0.4,
0.5,
0.6,
0.7,
0.8,
0.9,
],
))
scheduler.load_state_dict(state_dict)
# With float cycle_size
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler = LinearCyclicalScheduler(
optimizer,
"lr",
start_value=1.2,
end_value=0.2,
cycle_size=10.00000012,
cycle_mult=1.0,
)
state_dict = scheduler.state_dict()
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run([0] * 9, max_epochs=2)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.2,
1.0,
0.8,
0.6,
0.4,
0.2,
0.4,
0.6,
0.8,
1.0,
# Cycle 2
1.2,
1.0,
0.8,
0.6,
0.4,
0.2,
0.4,
0.6, # 0.8, 1.0,
],
))
scheduler.load_state_dict(state_dict)
def test_param_group_scheduler_asserts():
t1 = torch.zeros([1], requires_grad=True)
t2 = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([
{
"params": t1,
"lr": 0.1
},
{
"params": t2,
"lr": 0.1
},
])
lr_scheduler1 = LinearCyclicalScheduler(
optimizer,
"lr",
param_group_index=0,
start_value=1.0,
end_value=0.0,
cycle_size=10,
)
lr_scheduler2 = LinearCyclicalScheduler(
optimizer,
"lr",
param_group_index=1,
start_value=1.0,
end_value=0.0,
cycle_size=10,
)
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[0, 1, 2], names=["a", "b", "c"])
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, "2"], names=["a", "b"])
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names="ab")
with pytest.raises(ValueError):
ParamGroupScheduler(
schedulers=[lr_scheduler1, lr_scheduler2],
names=[
"a",
],
)
scheduler = ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names=["a", "b"])
with pytest.raises(
ValueError,
match=
r"Required state attribute 'schedulers' is absent in provided state_dict",
):
scheduler.load_state_dict({"a": 1})
with pytest.raises(
ValueError,
match=
r"Input state_dict contains 0 state_dicts of param group schedulers",
):
scheduler.load_state_dict({"schedulers": []})
with pytest.raises(
ValueError,
match=r"Name of scheduler from input state dict does not "
r"correspond to required one",
):
scheduler.load_state_dict({
"schedulers": [("a", lr_scheduler1.state_dict()), ("bad_name", {})]
})
def test_param_group_scheduler_asserts():
t1 = torch.zeros([1], requires_grad=True)
t2 = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([
{
"params": t1,
'lr': 0.1
},
{
"params": t2,
'lr': 0.1
},
])
lr_scheduler1 = LinearCyclicalScheduler(optimizer,
"lr",
param_group_index=0,
start_value=1.0,
end_value=0.0,
cycle_size=10)
lr_scheduler2 = LinearCyclicalScheduler(optimizer,
"lr",
param_group_index=1,
start_value=1.0,
end_value=0.0,
cycle_size=10)
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[0, 1, 2], names=['a', 'b', 'c'])
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, '2'], names=['a', 'b'])
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names='ab')
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names=[
'a',
])
scheduler = ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names=['a', 'b'])
with pytest.raises(
ValueError,
match=
r"Required state attribute 'schedulers' is absent in provided state_dict"
):
scheduler.load_state_dict({'a': 1})
with pytest.raises(
ValueError,
match=
r"Input state_dict contains 0 state_dicts of param group schedulers"
):
scheduler.load_state_dict({'schedulers': []})
with pytest.raises(
ValueError,
match=r"Name of scheduler from input state dict does not "
r"correspond to required one"):
scheduler.load_state_dict({
'schedulers': [('a', lr_scheduler1.state_dict()), ('bad_name', {})]
})