forked from rwth-i6/returnn
/
LearningRateControl.py
483 lines (432 loc) · 17.2 KB
/
LearningRateControl.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
import os
from Util import betterRepr, simpleObjRepr, ObjAsDict
from Log import log
import numpy
class LearningRateControl(object):
need_error_info = True
class EpochData:
def __init__(self, learningRate, error=None):
"""
:type learningRate: float
:type error: dict[str,float] | None
"""
self.learningRate = learningRate
if isinstance(error, float): # Old format.
error = {"old_format_score": error}
if error is None:
error = {}
self.error = error
__repr__ = simpleObjRepr
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
return {
"defaultLearningRate": config.float('learning_rate', 1.0),
"minLearningRate": config.float('min_learning_rate', 0.0),
"defaultLearningRates": config.typed_value('learning_rates') or config.float_list('learning_rates'),
"errorMeasureKey": config.value('learning_rate_control_error_measure', None),
"relativeErrorAlsoRelativeToLearningRate": config.bool('learning_rate_control_relative_error_relative_lr', False),
"filename": config.value('learning_rate_file', None),
}
@classmethod
def load_initial_from_config(cls, config):
"""
:type config: Config.Config
:rtype: LearningRateControl
"""
kwargs = cls.load_initial_kwargs_from_config(config)
return cls(**kwargs)
def __init__(self, defaultLearningRate, minLearningRate=0.0, defaultLearningRates=None,
errorMeasureKey=None,
relativeErrorAlsoRelativeToLearningRate=False,
filename=None):
"""
:param float defaultLearningRate: default learning rate. usually for epoch 1
:param list[float] | dict[int,float] defaultLearningRates: learning rates
:param str errorMeasureKey: for getEpochErrorValue() the selector for EpochData.error which is a dict
:param str filename: load from and save to file
"""
self.epochData = {} # type: dict[int,LearningRateControl.EpochData]
self.defaultLearningRate = defaultLearningRate
self.minLearningRate = minLearningRate
if defaultLearningRates:
if isinstance(defaultLearningRates, list):
defaultLearningRates = {i + 1: v for (i, v) in enumerate(defaultLearningRates)}
if isinstance(defaultLearningRates, (str, unicode)):
defaultLearningRates = eval(defaultLearningRates)
assert isinstance(defaultLearningRates, dict)
for epoch, v in defaultLearningRates.items():
self.setDefaultLearningRateForEpoch(epoch, v)
self.defaultLearningRates = defaultLearningRates
self.errorMeasureKey = errorMeasureKey
self.relativeErrorAlsoRelativeToLearningRate = relativeErrorAlsoRelativeToLearningRate
self.filename = filename
if filename:
if os.path.exists(filename):
print >> log.v4, "Learning-rate-control: loading file %s" % filename
self.load()
else:
print >> log.v4, "Learning-rate-control: file %s does not exist yet" % filename
else:
print >> log.v4, "Learning-rate-control: no file specified, not saving history (no proper restart possible)"
__repr__ = simpleObjRepr
def __str__(self):
return "%r, epoch data: %s" % \
(self, ", ".join(["%i: %s" % (epoch, self.epochData[epoch])
for epoch in sorted(self.epochData.keys())]))
def calcLearningRateForEpoch(self, epoch):
"""
:type epoch: int
:returns learning rate
:rtype: float
"""
raise NotImplementedError
def getLearningRateForEpoch(self, epoch):
"""
:type epoch: int
:rtype: float
"""
assert epoch >= 1
if epoch in self.epochData: return self.epochData[epoch].learningRate
learningRate = max(self.calcLearningRateForEpoch(epoch), self.minLearningRate)
self.setDefaultLearningRateForEpoch(epoch, learningRate)
return learningRate
def setDefaultLearningRateForEpoch(self, epoch, learningRate):
"""
:type epoch: int
:type learningRate: float
"""
if epoch in self.epochData:
if not self.epochData[epoch].learningRate:
self.epochData[epoch].learningRate = learningRate
else:
self.epochData[epoch] = self.EpochData(learningRate)
def getLastEpoch(self, epoch):
epochs = sorted([e for e in self.epochData.keys() if e < epoch])
if not epochs:
return None
return epochs[-1]
def getMostRecentLearningRate(self, epoch, excludeCurrent=True):
for e, data in reversed(sorted(self.epochData.items())):
if e > epoch: continue
if excludeCurrent and e == epoch: continue
if data.learningRate is None: continue
return data.learningRate
return self.defaultLearningRate
def calcRelativeError(self, oldEpoch, newEpoch):
oldError = self.getEpochErrorValue(oldEpoch)
newError = self.getEpochErrorValue(newEpoch)
if oldError is None or newError is None:
return None
relativeError = (newError - oldError) / abs(newError)
if self.relativeErrorAlsoRelativeToLearningRate:
learningRate = self.getMostRecentLearningRate(newEpoch, excludeCurrent=False)
# If the learning rate is lower than the initial learning rate,
# the relative error is also expected to be lower, so correct for that here.
relativeError /= learningRate / self.defaultLearningRate
return relativeError
def setEpochError(self, epoch, error):
"""
:type epoch: int
:type error: dict[str,float|dict[str,float]]
"""
if epoch not in self.epochData:
print >> log.v4, "Learning rate not set for epoch %i. Assuming default." % epoch
self.getLearningRateForEpoch(epoch) # This will set it.
assert isinstance(error, dict)
error = error.copy()
for k, v in list(error.items()):
if isinstance(v, dict): # like error = {"dev_score": {"cost:output1": .., "cost:output2": ...}, ...}
del error[k]
if len(v) == 1:
error[k] = v.values()[0]
continue
for k1, v1 in v.items():
if ":" in k1: k1 = k1[k1.index(":") + 1:]
error[k + "_" + k1] = v1
for v in error.values():
assert isinstance(v, float)
self.epochData[epoch].error.update(error)
def getErrorKey(self, epoch):
if epoch not in self.epochData:
return self.errorMeasureKey
epoch_data = self.epochData[epoch]
if not epoch_data.error:
return None
if len(epoch_data.error) == 1 and "old_format_score" in epoch_data.error:
return "old_format_score"
if self.errorMeasureKey:
if self.errorMeasureKey not in epoch_data.error:
if self.errorMeasureKey + "_output" in epoch_data.error: # for multiple outputs, try default output
return self.errorMeasureKey + "_output"
return self.errorMeasureKey
for key in ["dev_score", "train_score"]: # To keep old setups producing the same behavior, keep this order.
if key in epoch_data.error:
return key
return min(epoch_data.error.keys())
def getEpochErrorDict(self, epoch):
if epoch not in self.epochData:
return {}
return self.epochData[epoch].error
def getEpochErrorValue(self, epoch):
error = self.getEpochErrorDict(epoch)
if not error:
return None
key = self.getErrorKey(epoch)
assert key
assert key in error, "%r not in %r. fix %r in config. set it to %r or so." % \
(key, error, 'learning_rate_control_error_measure', 'dev_error')
return error[key]
def save(self):
if not self.filename: return
# First write to a temp-file, to be sure that the write happens without errors.
# Otherwise, it could happen that we delete the old existing file, then
# some error happens (e.g. disk quota), and we loose the newbob data.
# Loosing that data is very bad because it basically means that we have to redo all the training.
tmp_filename = self.filename + ".new_tmp"
f = open(tmp_filename, "w")
f.write(betterRepr(self.epochData))
f.write("\n")
f.close()
os.rename(tmp_filename, self.filename)
def load(self):
s = open(self.filename).read()
self.epochData = eval(s, {"nan": float("nan")}, ObjAsDict(self))
class ConstantLearningRate(LearningRateControl):
need_error_info = False
def calcLearningRateForEpoch(self, epoch):
"""
Dummy constant learning rate. Returns initial learning rate.
:type epoch: int
:returns learning rate
:rtype: float
"""
while True:
lastEpoch = self.getLastEpoch(epoch)
if lastEpoch is None:
return self.defaultLearningRate
learningRate = self.epochData[lastEpoch].learningRate
if learningRate is None:
epoch = lastEpoch
continue
return learningRate
class NewbobRelative(LearningRateControl):
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
kwargs = super(NewbobRelative, cls).load_initial_kwargs_from_config(config)
kwargs.update({
"relativeErrorThreshold": config.float('newbob_relative_error_threshold', -0.01),
"learningRateDecayFactor": config.float('newbob_learning_rate_decay', 0.5)})
return kwargs
def __init__(self, relativeErrorThreshold, learningRateDecayFactor, **kwargs):
"""
:param float defaultLearningRate: learning rate for epoch 1+2
:type relativeErrorThreshold: float
:type learningRateDecayFactor: float
:type filename: str
"""
super(NewbobRelative, self).__init__(**kwargs)
self.relativeErrorThreshold = relativeErrorThreshold
self.learningRateDecayFactor = learningRateDecayFactor
def calcLearningRateForEpoch(self, epoch):
"""
Newbob+ on train data.
:type epoch: int
:returns learning rate
:rtype: float
"""
lastEpoch = self.getLastEpoch(epoch)
if lastEpoch is None:
return self.defaultLearningRate
learningRate = self.epochData[lastEpoch].learningRate
if learningRate is None:
return self.defaultLearningRate
last2Epoch = self.getLastEpoch(lastEpoch)
if last2Epoch is None:
return learningRate
relativeError = self.calcRelativeError(last2Epoch, lastEpoch)
if relativeError is None:
return learningRate
if relativeError > self.relativeErrorThreshold:
learningRate *= self.learningRateDecayFactor
return learningRate
class NewbobAbs(LearningRateControl):
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
kwargs = super(NewbobAbs, cls).load_initial_kwargs_from_config(config)
kwargs.update({
"errorThreshold": config.float('newbob_error_threshold', -0.01),
"learningRateDecayFactor": config.float('newbob_learning_rate_decay', 0.5)})
return kwargs
def __init__(self, errorThreshold, learningRateDecayFactor, **kwargs):
"""
:type errorThreshold: float
:type learningRateDecayFactor: float
"""
super(NewbobAbs, self).__init__(**kwargs)
self.errorThreshold = errorThreshold
self.learningRateDecayFactor = learningRateDecayFactor
def calcLearningRateForEpoch(self, epoch):
"""
Newbob+ on train data.
:type epoch: int
:returns learning rate
:rtype: float
"""
lastEpoch = self.getLastEpoch(epoch)
if lastEpoch is None:
return self.defaultLearningRate
learningRate = self.epochData[lastEpoch].learningRate
if learningRate is None:
return self.defaultLearningRate
last2Epoch = self.getLastEpoch(lastEpoch)
if last2Epoch is None:
return learningRate
oldError = self.getEpochErrorValue(last2Epoch)
newError = self.getEpochErrorValue(lastEpoch)
if oldError is None or newError is None:
return learningRate
errorDiff = newError - oldError
if errorDiff > self.errorThreshold:
learningRate *= self.learningRateDecayFactor
return learningRate
class NewbobMultiEpoch(LearningRateControl):
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
kwargs = super(NewbobMultiEpoch, cls).load_initial_kwargs_from_config(config)
kwargs.update({
"numEpochs": config.int("newbob_multi_num_epochs", 5),
"updateInterval": config.int("newbob_multi_update_interval", config.int("newbob_multi_num_epochs", 5)),
"relativeErrorThreshold": config.float('newbob_relative_error_threshold', -0.01),
"learningRateDecayFactor": config.float('newbob_learning_rate_decay', 0.5)})
return kwargs
def __init__(self, numEpochs, updateInterval, relativeErrorThreshold, learningRateDecayFactor, **kwargs):
"""
:param float defaultLearningRate: learning rate for epoch 1+2
:type numEpochs: int
:type updateInterval: int
:type relativeErrorThreshold: float
:type learningRateDecayFactor: float
:type filename: str
"""
super(NewbobMultiEpoch, self).__init__(**kwargs)
self.numEpochs = numEpochs
assert self.numEpochs >= 1
self.updateInterval = updateInterval
assert self.updateInterval >= 1
self.relativeErrorThreshold = relativeErrorThreshold
self.learningRateDecayFactor = learningRateDecayFactor
def _calcMeanRelativeError(self, epochs):
assert len(epochs) >= 2
errors = [self.calcRelativeError(epochs[i], epochs[i + 1]) for i in range(len(epochs) - 1)]
if any([e is None for e in errors]):
return None
return numpy.mean(errors)
def calcLearningRateForEpoch(self, epoch):
"""
Newbob+ on train data.
:type epoch: int
:returns learning rate
:rtype: float
"""
learningRate = self.getMostRecentLearningRate(epoch)
lastEpochs = sorted([e for e in self.epochData.keys() if e < epoch])
if not lastEpochs:
return learningRate
# We could also use the self.numEpochs limit here. But maybe this is better.
if len(lastEpochs) <= 1:
return learningRate
# We start counting epochs at 1.
if self.updateInterval > 1 and epoch % self.updateInterval != 1:
return learningRate
# Take one more than self.numEpochs because we are looking at the diffs.
lastEpochs = lastEpochs[-self.numEpochs - 1:]
meanRelativeError = self._calcMeanRelativeError(lastEpochs)
if meanRelativeError > self.relativeErrorThreshold:
learningRate *= self.learningRateDecayFactor
return learningRate
def learningRateControlType(typeName):
if typeName == "constant":
return ConstantLearningRate
elif typeName in ("newbob", "newbob_rel", "newbob_relative"): # Old setups expect the relative version.
return NewbobRelative
elif typeName == "newbob_abs":
return NewbobAbs
elif typeName == "newbob_multi_epoch":
return NewbobMultiEpoch
else:
assert False, "unknown learning-rate-control type %s" % typeName
def loadLearningRateControlFromConfig(config):
"""
:type config: Config.Config
:rtype: LearningRateControl
"""
controlType = config.value("learning_rate_control", "constant")
cls = learningRateControlType(controlType)
return cls.load_initial_from_config(config)
def demo():
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options]" % __file__)
print("example usage: python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001" % __file__)
rnn.initConfig(commandLineOptions=sys.argv[1:])
from Pretrain import pretrainFromConfig
pretrain = pretrainFromConfig(rnn.config)
first_non_pretrain_epoch = 1
pretrain_learning_rate = None
if pretrain:
first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1
rnn.config._hack_value_reading_debug()
log.initialize(verbosity=[5])
control = loadLearningRateControlFromConfig(rnn.config)
print("LearningRateControl: %r" % control)
if not control.epochData:
print("No epoch data so far.")
return
if pretrain:
pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.defaultLearningRate)
maxEpoch = max(control.epochData.keys())
for epoch in range(1, maxEpoch + 2): # all epochs [1..maxEpoch+1]
oldLearningRate = None
if epoch in control.epochData:
oldLearningRate = control.epochData[epoch].learningRate
if epoch < first_non_pretrain_epoch:
learningRate = pretrain_learning_rate
s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learningRate, oldLearningRate)
elif first_non_pretrain_epoch > 1 and epoch == first_non_pretrain_epoch:
learningRate = control.defaultLearningRate
s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % (epoch, learningRate, oldLearningRate)
else:
learningRate = control.calcLearningRateForEpoch(epoch)
s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learningRate, oldLearningRate)
if learningRate < control.minLearningRate:
learningRate = control.minLearningRate
s += ", clipped to %s" % learningRate
s += ", previous relative error: %s" % control.calcRelativeError(epoch - 2, epoch - 1)
print(s)
# Overwrite new learning rate so that the calculation for further learning rates stays consistent.
if epoch in control.epochData:
control.epochData[epoch].learningRate = learningRate
else:
control.epochData[epoch] = control.EpochData(learningRate=learningRate)
print("Finished, last stored epoch was %i." % maxEpoch)
if __name__ == "__main__":
demo()