def test_concat_scheduler_3_schedulers():
tensor = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler_1 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=1.0,
end_value=0.5,
cycle_size=20)
scheduler_2 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=0.5,
end_value=0.45,
cycle_size=10)
scheduler_3 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=0.5,
end_value=0.0,
cycle_size=20)
durations = [10, 5]
concat_scheduler = ConcatScheduler(
schedulers=[scheduler_1, scheduler_2, scheduler_3],
durations=durations,
save_history=True)
state_dict = concat_scheduler.state_dict()
data = [0] * 10
max_epochs = 2
simulated_values = ConcatScheduler.simulate_values(
num_events=len(data) * max_epochs,
schedulers=[scheduler_1, scheduler_2, scheduler_3],
durations=durations)
def save_lr(engine):
lrs.append(optimizer.param_groups[0]['lr'])
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, concat_scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run(data, max_epochs=max_epochs)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1 of the first LinearCyclicalScheduler
1.0,
0.95,
0.9,
0.85,
0.8,
0.75,
0.7,
0.65,
0.6,
0.55,
# Cycle 1 of the second LinearCyclicalScheduler
0.5,
0.49,
0.48,
0.47,
0.46,
# Cycle 1 of the third LinearCyclicalScheduler
0.5,
0.45,
0.4,
0.35,
0.3,
]))
state_lrs = trainer.state.param_history['lr']
assert len(state_lrs) == len(lrs)
# Unpack singleton lists
assert [group[0] for group in state_lrs] == lrs
assert lrs == pytest.approx([v for i, v in simulated_values])
concat_scheduler.load_state_dict(state_dict)
def _test(duration_vals_as_np_int):
scheduler_1 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=1.0,
end_value=0.0,
cycle_size=10)
scheduler_2 = CosineAnnealingScheduler(optimizer,
"lr",
start_value=0.0,
end_value=1.0,
cycle_size=10)
durations = [
10,
]
if duration_vals_as_np_int:
durations = [np.int64(t) for t in durations]
concat_scheduler = ConcatScheduler(
schedulers=[scheduler_1, scheduler_2],
durations=durations,
save_history=True)
state_dict = concat_scheduler.state_dict()
data = [0] * 10
max_epochs = 2
simulated_values = ConcatScheduler.simulate_values(
num_events=len(data) * max_epochs,
schedulers=[scheduler_1, scheduler_2],
durations=durations)
def save_lr(engine):
lrs.append(optimizer.param_groups[0]['lr'])
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, concat_scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run(data, max_epochs=max_epochs)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1 of the LinearCyclicalScheduler
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
# Cycle 1 of the CosineAnnealingScheduler
0.0,
0.02447174185242318,
0.09549150281252627,
0.20610737385376332,
0.3454915028125263,
0.5,
0.6545084971874737,
0.7938926261462365,
0.9045084971874737,
0.9755282581475768,
]))
state_lrs = trainer.state.param_history['lr']
assert len(state_lrs) == len(lrs)
# Unpack singleton lists
assert [group[0] for group in state_lrs] == lrs
assert lrs == pytest.approx([v for i, v in simulated_values])
concat_scheduler.load_state_dict(state_dict)
def test_concat_scheduler_two_linear():
tensor = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler_1 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=0.0,
end_value=0.1,
cycle_size=2)
scheduler_2 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=0.2,
end_value=1.0,
cycle_size=2)
durations = [
5,
]
concat_scheduler = ConcatScheduler(schedulers=[scheduler_1, scheduler_2],
durations=durations,
save_history=True)
state_dict = concat_scheduler.state_dict()
assert concat_scheduler.get_param() == 0.0
data = [0] * 10
max_epochs = 2
simulated_values = ConcatScheduler.simulate_values(
num_events=len(data) * max_epochs,
schedulers=[scheduler_1, scheduler_2],
durations=durations)
def save_lr(engine):
lrs.append(optimizer.param_groups[0]['lr'])
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, concat_scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run(data, max_epochs=max_epochs)
assert lrs == list(
map(
pytest.approx,
[
# first LinearCyclicalScheduler
0.0,
0.1,
0.0,
0.1,
0.0,
# second LinearCyclicalScheduler
0.2,
1.0,
0.2,
1.0,
0.2,
1.0,
0.2,
1.0,
0.2,
1.0,
0.2,
1.0,
0.2,
1.0,
0.2,
]))
state_lrs = trainer.state.param_history['lr']
assert len(state_lrs) == len(lrs)
# Unpack singleton lists
assert [group[0] for group in state_lrs] == lrs
assert lrs == pytest.approx([v for i, v in simulated_values])
concat_scheduler.load_state_dict(state_dict)
def test_param_group_scheduler_asserts():
t1 = torch.zeros([1], requires_grad=True)
t2 = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([
{
"params": t1,
'lr': 0.1
},
{
"params": t2,
'lr': 0.1
},
])
lr_scheduler1 = LinearCyclicalScheduler(optimizer,
"lr",
param_group_index=0,
start_value=1.0,
end_value=0.0,
cycle_size=10)
lr_scheduler2 = LinearCyclicalScheduler(optimizer,
"lr",
param_group_index=1,
start_value=1.0,
end_value=0.0,
cycle_size=10)
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[0, 1, 2], names=['a', 'b', 'c'])
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, '2'], names=['a', 'b'])
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names='ab')
with pytest.raises(ValueError):
ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names=[
'a',
])
scheduler = ParamGroupScheduler(schedulers=[lr_scheduler1, lr_scheduler2],
names=['a', 'b'])
with pytest.raises(
ValueError,
match=
r"Required state attribute 'schedulers' is absent in provided state_dict"
):
scheduler.load_state_dict({'a': 1})
with pytest.raises(
ValueError,
match=
r"Input state_dict contains 0 state_dicts of param group schedulers"
):
scheduler.load_state_dict({'schedulers': []})
with pytest.raises(
ValueError,
match=r"Name of scheduler from input state dict does not "
r"correspond to required one"):
scheduler.load_state_dict({
'schedulers': [('a', lr_scheduler1.state_dict()), ('bad_name', {})]
})
def test_linear_scheduler():
with pytest.raises(
TypeError,
match=r"Argument optimizer should be torch.optim.Optimizer"):
LinearCyclicalScheduler({}, 'lr', 1, 0, cycle_size=0)
tensor = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0.0)
with pytest.raises(
ValueError,
match=r"Argument cycle_size should be positive and larger than 1"):
LinearCyclicalScheduler(optimizer, 'lr', 1, 0, cycle_size=0)
with pytest.raises(
ValueError,
match=r"Argument cycle_size should be positive and larger than 1"):
LinearCyclicalScheduler(optimizer, 'lr', 1, 0, cycle_size=1)
scheduler = LinearCyclicalScheduler(optimizer, 'lr', 1, 0, 10)
state_dict = scheduler.state_dict()
def save_lr(engine):
lrs.append(optimizer.param_groups[0]['lr'])
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run([0] * 9, max_epochs=2)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
# Cycle 2
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4, # 0.6, 0.8,
]))
scheduler.load_state_dict(state_dict)
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler = LinearCyclicalScheduler(optimizer,
'lr',
1,
0,
10,
cycle_mult=2)
state_dict = scheduler.state_dict()
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run([0] * 10, max_epochs=3)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
# Cycle 2
1.0,
0.9,
0.8,
0.7,
0.6,
0.5,
0.4,
0.3,
0.2,
0.1,
0.0,
0.1,
0.2,
0.3,
0.4,
0.5,
0.6,
0.7,
0.8,
0.9,
]))
scheduler.load_state_dict(state_dict)
# With float cycle_size
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler = LinearCyclicalScheduler(optimizer,
'lr',
start_value=1.2,
end_value=0.2,
cycle_size=10.00000012,
cycle_mult=1.0)
state_dict = scheduler.state_dict()
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
for _ in range(2):
lrs = []
trainer.run([0] * 9, max_epochs=2)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.2,
1.0,
0.8,
0.6,
0.4,
0.2,
0.4,
0.6,
0.8,
1.0,
# Cycle 2
1.2,
1.0,
0.8,
0.6,
0.4,
0.2,
0.4,
0.6, # 0.8, 1.0,
]))
scheduler.load_state_dict(state_dict)
def _test(save_history):
tensor = torch.ones([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0.001)
max_epochs = 25
lr_max_value = 0.4
num_iterations_per_epoch = 128
num_iterations = max_epochs * num_iterations_per_epoch
warmup_duration = 5 * num_iterations_per_epoch
cooldown_duration = 5 * num_iterations_per_epoch
scheduler_1 = LinearCyclicalScheduler(
optimizer,
"lr",
start_value=lr_max_value,
end_value=lr_max_value * 0.9,
cycle_size=(num_iterations - warmup_duration - cooldown_duration) *
2)
scheduler_2 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=lr_max_value,
end_value=0.0,
cycle_size=cooldown_duration * 2)
lr_scheduler = ConcatScheduler(schedulers=[scheduler_1, scheduler_2],
durations=[
num_iterations - warmup_duration -
cooldown_duration,
],
save_history=False)
lr_values = [None] * num_iterations
scheduler = create_lr_scheduler_with_warmup(
lr_scheduler,
warmup_start_value=0.0,
warmup_end_value=lr_max_value,
warmup_duration=warmup_duration,
save_history=save_history,
output_simulated_values=lr_values)
state_dict = scheduler.state_dict()
trainer = Engine(lambda engine, batch: None)
@trainer.on(Events.ITERATION_COMPLETED)
def save_lr(engine):
lrs.append(optimizer.param_groups[0]['lr'])
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
data = [0] * num_iterations_per_epoch
for _ in range(2):
lrs = []
trainer.run(data, max_epochs=max_epochs)
assert lrs == pytest.approx([v for i, v in lr_values])
if save_history:
param_history = trainer.state.param_history['lr']
assert lrs == pytest.approx([v[0] for v in param_history])
scheduler.load_state_dict(state_dict)
def test_create_lr_scheduler_with_warmup():
with pytest.raises(TypeError, match=r"Argument lr_scheduler should be a subclass of"):
create_lr_scheduler_with_warmup(12, warmup_start_value=0.0, warmup_end_value=0.1, warmup_duration=10)
t1 = torch.zeros([1], requires_grad=True)
# A) opt lr != warmup_end_value
optimizer = torch.optim.SGD([t1], lr=0.2)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.98)
with pytest.raises(ValueError, match=r"Argument warmup_duration should be at least 2 events"):
create_lr_scheduler_with_warmup(
torch_lr_scheduler, warmup_start_value=0.0, warmup_end_value=0.1, warmup_duration=1
)
with pytest.raises(ValueError, match=r"Argument warmup_duration should be at least 2 events"):
create_lr_scheduler_with_warmup(
torch_lr_scheduler, warmup_start_value=0.0, warmup_end_value=0.1, warmup_duration="abc"
)
with pytest.raises(TypeError, match=r"Argument output_simulated_values should be a list of None"):
simulated_values = ()
create_lr_scheduler_with_warmup(
torch_lr_scheduler,
warmup_start_value=0.0,
warmup_end_value=0.1,
warmup_duration=10,
output_simulated_values=simulated_values,
)
def _test(lr_scheduler, optimizer, warmup_start_value, warmup_end_value, warmup_duration, warmup_end_next_value):
num_iterations = 10
max_epochs = 20
simulated_values = [None] * (num_iterations * max_epochs)
scheduler = create_lr_scheduler_with_warmup(
lr_scheduler,
warmup_start_value=warmup_start_value,
warmup_end_value=warmup_end_value,
warmup_duration=warmup_duration,
output_simulated_values=simulated_values,
)
if warmup_end_value is None:
warmup_end_value = optimizer.param_groups[0]["lr"]
state_dict = scheduler.state_dict()
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
@trainer.on(Events.ITERATION_STARTED)
def save_lr(engine):
lrs.append(optimizer.param_groups[0]["lr"])
data = [0] * num_iterations
for _ in range(2):
lrs = []
trainer.run(data, max_epochs=max_epochs)
assert lrs == pytest.approx([v for i, v in simulated_values])
assert lrs[0] == pytest.approx(warmup_start_value), "lrs={}".format(lrs[: warmup_duration + num_iterations])
assert lrs[warmup_duration - 1] == pytest.approx(warmup_end_value), "lrs={}".format(
lrs[: warmup_duration + num_iterations]
)
assert lrs[warmup_duration] == pytest.approx(warmup_end_next_value), "lrs={}".format(
lrs[: warmup_duration + num_iterations]
)
scheduler.load_state_dict(state_dict)
t1 = torch.zeros([1], requires_grad=True)
# A) opt lr != warmup_end_value
optimizer = torch.optim.SGD([t1], lr=0.2)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.98)
_test(torch_lr_scheduler, optimizer, 0.01, 0.05, 10, 0.2)
optimizer = torch.optim.SGD([t1], lr=0.2)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.98)
_test(torch_lr_scheduler, optimizer, 0.01, 0.05, 2, 0.2)
# B) opt lr == warmup_end_value
optimizer = torch.optim.SGD([t1], lr=0.2)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.98)
_test(torch_lr_scheduler, optimizer, 0.01, 0.2, 10, 0.2 * 0.98)
optimizer = torch.optim.SGD([t1], lr=0.2)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.98)
_test(torch_lr_scheduler, optimizer, 0.01, 0.2, 2, 0.2 * 0.98)
# C) lr_scheduler start_value != warmup_end_value
t1 = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([t1], lr=0.0)
lr_scheduler = LinearCyclicalScheduler(
optimizer=optimizer, param_name="lr", start_value=0.8, end_value=0.0, cycle_size=10
)
_test(lr_scheduler, optimizer, 0.01, 0.05, 10, 0.8)
optimizer = torch.optim.SGD([t1], lr=0.0)
lr_scheduler = LinearCyclicalScheduler(
optimizer=optimizer, param_name="lr", start_value=0.8, end_value=0.0, cycle_size=10
)
_test(lr_scheduler, optimizer, 0.01, 0.05, 2, 0.8)
# D) lr_scheduler start_value == warmup_end_value
t1 = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([t1], lr=0.0)
lr_scheduler = LinearCyclicalScheduler(
optimizer=optimizer, param_name="lr", start_value=0.8, end_value=0.0, cycle_size=10
)
_test(lr_scheduler, optimizer, 0.01, 0.8, 10, 0.8 - (0.8 / 5.0))
optimizer = torch.optim.SGD([t1], lr=0.0)
lr_scheduler = LinearCyclicalScheduler(
optimizer=optimizer, param_name="lr", start_value=0.8, end_value=0.0, cycle_size=10
)
_test(lr_scheduler, optimizer, 0.01, 0.8, 2, 0.8 - (0.8 / 5.0))
# E) warmup_end_value is None: fall back to case B)
optimizer = torch.optim.SGD([t1], lr=0.2)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.98)
_test(torch_lr_scheduler, optimizer, 0.01, None, 10, 0.2 * 0.98)
def test_simulate_and_plot_values():
import matplotlib
matplotlib.use("Agg")
def _test(scheduler_cls, **scheduler_kwargs):
optimizer = None
event = Events.ITERATION_STARTED
if scheduler_cls == LRScheduler:
scheduler_kwargs["optimizer"] = scheduler_kwargs["lr_scheduler"].optimizer
optimizer = scheduler_kwargs["optimizer"]
event = Events.ITERATION_COMPLETED
elif scheduler_cls == ConcatScheduler:
optimizer = scheduler_kwargs["optimizer"]
del scheduler_kwargs["optimizer"]
else:
tensor = torch.zeros([1], requires_grad=True)
scheduler_kwargs["optimizer"] = torch.optim.SGD([tensor], lr=0.1)
optimizer = scheduler_kwargs["optimizer"]
max_epochs = 2
data = [0] * 10
# simulated_values = scheduler_cls.simulate_values(num_events=len(data) * max_epochs, **scheduler_kwargs)
scheduler = scheduler_cls(**scheduler_kwargs)
lrs = []
def save_lr(engine):
lrs.append(optimizer.param_groups[0]["lr"])
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(event, scheduler)
trainer.add_event_handler(Events.ITERATION_STARTED, save_lr)
trainer.run(data, max_epochs=max_epochs)
# assert lrs == pytest.approx([v for i, v in simulated_values])
if scheduler_cls == LRScheduler or scheduler_cls == ConcatScheduler:
# As internal state of torch lr scheduler has been changed the following checks will fail
return
# reexecute to check if no internal changes
# simulated_values = scheduler_cls.simulate_values(num_events=len(data) * max_epochs,
# save_history=True, # this will be removed
# **scheduler_kwargs)
# assert lrs == pytest.approx([v for i, v in simulated_values])
# launch plot values
scheduler_cls.plot_values(num_events=len(data) * max_epochs, **scheduler_kwargs)
# LinearCyclicalScheduler
_test(LinearCyclicalScheduler, param_name="lr", start_value=1.0, end_value=0.0, cycle_size=10)
# CosineAnnealingScheduler
_test(CosineAnnealingScheduler, param_name="lr", start_value=1.0, end_value=0.0, cycle_size=10)
# LRScheduler
tensor = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0.1)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.5)
_test(LRScheduler, lr_scheduler=torch_lr_scheduler)
# ConcatScheduler = [LinearCyclicalScheduler, CosineAnnealingScheduler]
scheduler_1 = LinearCyclicalScheduler(optimizer, "lr", start_value=1.0, end_value=0.0, cycle_size=20)
scheduler_2 = CosineAnnealingScheduler(optimizer, "lr", start_value=0.0, end_value=1.0, cycle_size=10)
durations = [10]
_test(ConcatScheduler, optimizer=optimizer, schedulers=[scheduler_1, scheduler_2], durations=durations)
# ConcatScheduler = [LinearCyclicalScheduler, LRScheduler]
tensor = torch.ones([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0.001)
torch_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=1.5)
scheduler_1 = LRScheduler(torch_lr_scheduler)
scheduler_2 = LinearCyclicalScheduler(optimizer, "lr", start_value=0.1, end_value=0.0, cycle_size=10)
durations = [10]
_test(ConcatScheduler, optimizer=optimizer, schedulers=[scheduler_1, scheduler_2], durations=durations)
# PiecewiseLinear
tensor = torch.ones([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0.001)
_test(
PiecewiseLinear,
optimizer=optimizer,
param_name="lr",
milestones_values=[(10, 0.5), (20, 0.45), (21, 0.3), (30, 0.1), (40, 0.1)],
)
def test_linear_scheduler():
with pytest.raises(ValueError):
LinearCyclicalScheduler({}, 'lr', 1, 0, cycle_size=0)
with pytest.raises(ValueError):
LinearCyclicalScheduler({}, 'lr', 1, 0, cycle_size=1)
tensor = torch.zeros([1], requires_grad=True)
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler = LinearCyclicalScheduler(optimizer, 'lr', 1, 0, 10)
lrs = []
def save_lr(engine):
lrs.append(optimizer.param_groups[0]['lr'])
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
trainer.run([0] * 10, max_epochs=2)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
# Cycle 2
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
]))
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler = LinearCyclicalScheduler(optimizer,
'lr',
1,
0,
10,
cycle_mult=2)
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
lrs = []
trainer.run([0] * 10, max_epochs=3)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.0,
0.8,
0.6,
0.4,
0.2,
0.0,
0.2,
0.4,
0.6,
0.8,
# Cycle 2
1.0,
0.9,
0.8,
0.7,
0.6,
0.5,
0.4,
0.3,
0.2,
0.1,
0.0,
0.1,
0.2,
0.3,
0.4,
0.5,
0.6,
0.7,
0.8,
0.9,
]))
# With float cycle_size
optimizer = torch.optim.SGD([tensor], lr=0)
scheduler = LinearCyclicalScheduler(optimizer,
'lr',
start_value=1.2,
end_value=0.2,
cycle_size=10.00000012,
cycle_mult=1.0)
trainer = Engine(lambda engine, batch: None)
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
trainer.add_event_handler(Events.ITERATION_COMPLETED, save_lr)
lrs = []
trainer.run([0] * 10, max_epochs=2)
assert lrs == list(
map(
pytest.approx,
[
# Cycle 1
1.2,
1.0,
0.8,
0.6,
0.4,
0.2,
0.4,
0.6,
0.8,
1.0,
# Cycle 2
1.2,
1.0,
0.8,
0.6,
0.4,
0.2,
0.4,
0.6,
0.8,
1.0,
]))
training_saver = ModelCheckpoint(args.log_dir + '/checkpoints',
filename_prefix="checkpoint",
save_interval=1,
n_saved=1,
save_as_state_dict=False,
create_dir=True)
to_save = {
"network": model,
"model_state": model.state_dict(),
"optimizer_state": optimizer.state_dict()
}
scheduler_1 = LinearCyclicalScheduler(optimizer,
"lr",
start_value=1e-2,
end_value=1e-1,
cycle_size=60)
trainer.add_event_handler(Events.EPOCH_COMPLETED, training_saver, to_save)
#trainer.add_event_handler(Events.ITERATION_STARTED, scheduler_1)
@trainer.on(Events.EPOCH_COMPLETED)
def print_loss(engine):
print('Running Loss {:.2f}'.format(engine.state.metrics['smooth loss']))
print('L_BC {:.2f}'.format(engine.state.output[1].item()))
print('L_KL {:.2f}'.format(engine.state.output[-2].item()))
@trainer.on(Events.ITERATION_COMPLETED)
