def dump_iteration_state(self, main_loop):
if self.slim_iteration_state:
logger.info(" ...saving iteration state (slim)")
secure_dump(main_loop.epoch_iterator, self.path_to_iteration_state)
else:
logger.info(" ...saving iteration state (full)")
secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
def do(self, which_callback, *args):
current_value = self.main_loop.log.current_row.get(self.record_name)
if current_value is None:
return
best_value = self.main_loop.status.get(self.best_name, None)
if (best_value is None
or (current_value != best_value and self.choose_best(
current_value, best_value) == current_value)):
self.main_loop.status[self.best_name] = current_value
# save main_loop
_, from_user = self.parse_args(which_callback, args)
try:
path = self.path
if from_user:
path, = from_user
secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute),
filenames[attribute],
use_cpickle=self.use_cpickle)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(
SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path, ))
def do(self, callback_name, *args):
"""Pickle the save_separately parts (and not the main loop object) to disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
### this line is disabled from superclass impl to bypass using blocks.serialization.dump
### because pickling main thusly causes pickling error:
### "RuntimeError: maximum recursion depth exceeded while calling a Python object"
# secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute),
filenames[attribute], cPickle.dump)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path, ))
def do(self, callback_name, *args):
"""Pickle the save_separately parts (and not the main loop object) to disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
### this line is disabled from superclass impl to bypass using blocks.serialization.dump
### because pickling main thusly causes pickling error:
### "RuntimeError: maximum recursion depth exceeded while calling a Python object"
# secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute),
filenames[attribute], cPickle.dump)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path,))
def do(self, callback_name, *args):
"""Pickle the main loop object to the disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
to_add = None
if self.save_separately:
to_add = {attr: getattr(self.main_loop, attr) for attr in
self.save_separately}
if self.parameters is None:
if hasattr(self.main_loop, 'model'):
self.parameters = self.main_loop.model.parameters
object_ = None
if self.save_main_loop:
object_ = self.main_loop
secure_dump(object_, path,
dump_function=dump_and_add_to_dump,
parameters=self.parameters,
to_add=to_add,
use_cpickle=self.use_cpickle)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path,))
def do(self, callback_name, *args):
"""Pickle the main loop object to the disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute),
filenames[attribute], cPickle.dump)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path,))
def do_checkpoint(self, callback_name, *args):
logger.info("Checkpointing has started")
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
to_add = None
if self.save_separately:
to_add = {attr: getattr(self.main_loop, attr) for attr in
self.save_separately}
if self.parameters is None:
if hasattr(self.main_loop, 'model'):
self.parameters = self.main_loop.model.parameters
object_ = None
if self.save_main_loop:
object_ = self.main_loop
secure_dump(object_, path,
dump_function=dump_and_add_to_dump,
parameters=self.parameters,
to_add=to_add,
use_cpickle=self.use_cpickle)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path,))
logger.info("Checkpointing has finished")
def _dump(self):
try:
self.main_loop.log.current_row['saved_best_to'] = self.path
secure_dump(self.main_loop, self.path)
except Exception:
self.main_loop.log.current_row['saved_best_to'] = None
raise
def _dump(self):
try:
self.main_loop.log.current_row['saved_best_to'] = self.path
secure_dump(self.main_loop, self.path)
except Exception:
self.main_loop.log.current_row['saved_best_to'] = None
raise
def do(self, callback_name, *args):
"""Pickle the main loop object to the disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
secure_dump(self.get_save(), path, use_cpickle=self.use_cpickle)
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute),
filenames[attribute],
cPickle.dump,
protocol=DEFAULT_PROTOCOL)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path, ))
def test_secure_dump():
foo = object()
bar = lambda: None # flake8: noqa
with NamedTemporaryFile(delete=False, dir=config.temp_dir) as f:
secure_dump(foo, f.name)
assert_raises(PicklingError, secure_dump, bar, f.name)
with open(f.name, 'rb') as f:
assert type(load(f)) is object
def test_secure_dump():
foo = object()
bar = lambda: None # flake8: noqa
with NamedTemporaryFile(delete=False) as f:
secure_dump(foo, f.name)
assert_raises(PicklingError, secure_dump, bar, f.name)
with open(f.name, 'rb') as f:
assert type(load(f)) is object
def dump_iteration_state(self, main_loop):
if self.slim_iteration_state:
logger.info(" ...saving iteration state (slim)")
secure_dump(main_loop.epoch_iterator, self.path_to_iteration_state)
else:
logger.info(" ...saving iteration state (full)")
secure_dump(main_loop.iteration_state,
self.path_to_iteration_state)
def do(self, callback_name, *args):
"""Pickle the main loop object to the disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
sharedData = {}
_, from_user = self.parse_args(callback_name, args)
try:
#shared data manipulation
#add default values to temp share
for var in self.main_loop.model.variables:
if var.name != None and "sharedData" in var.name:
sharedData[var.name] = var.get_value(borrow=True)
if theano.config.floatX in str(var.type):
self.tempIndices[theano.config.floatX] += 1
temp = self.tempShared[theano.config.floatX][
self.tempIndices[theano.config.floatX]]
elif 'uint8' in str(var.type):
self.tempIndices['uint8'] += 1
temp = self.tempShared['uint8'][
self.tempIndices['uint8']]
else:
print(
'you have not accounted for all shared variables')
sys.exit(0)
var.set_value(temp.get_value(borrow=True), borrow=True)
path = self.path
if from_user:
path, = from_user
secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute),
filenames[attribute],
cPickle.dump,
protocol=DEFAULT_PROTOCOL)
except Exception:
path = None
raise
finally:
self.updateIndices()
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path, ))
# Change the shared variable back.
for var in self.main_loop.model.variables:
if var.name != None and "sharedData" in var.name and var.name in sharedData:
var.set_value(sharedData[var.name], borrow=True)
def _dump(self):
try:
path = self.path + '/best'
self.main_loop.log.current_row['saved_best_to'] = path
logger.info("Saving log ...")
f = open(self.path + '/log.txt', 'w')
f.write(str(self.main_loop.log))
f.close()
logger.info("Dumping best model ...")
secure_dump(self.main_loop.model.params, path, use_cpickle=True)
except Exception:
self.main_loop.log.current_row['saved_best_to'] = None
raise
def _dump(self):
try:
path = self.path + '/best'
self.main_loop.log.current_row['saved_best_to'] = path
logger.info("Saving log ...")
f = open(self.path + '/log.txt', 'w')
f.write(str(self.main_loop.log))
f.close()
logger.info("Dumping best model ...")
secure_dump(self.main_loop.model.params, path, use_cpickle=True)
except Exception:
self.main_loop.log.current_row['saved_best_to'] = None
raise
def do(self, callback_name, *args):
if self.notification_name in self.main_loop.log.current_row:
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute), filenames[attribute] + '.pkl', pickle.dump, protocol=pickle.HIGHEST_PROTOCOL)
# self.iteration += 1
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path,))
def do(self, callback_name, *args):
"""Pickle the main loop object to the disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
_, from_user = self.parse_args(callback_name, args)
if os.path.isfile(self.path):
count = int(self.path.split('.')[-1])
count += 1
self.path = ".".join(self.path.split('.')[:-1]) + "." + str(count)
try:
path = self.path
if from_user:
path, = from_user
to_add = None
if self.save_separately:
to_add = {
attr: getattr(self.main_loop, attr)
for attr in self.save_separately
}
if self.parameters is None:
if hasattr(self.main_loop, 'model'):
self.parameters = self.main_loop.model.parameters
object_ = None
if self.save_main_loop:
object_ = self.main_loop
secure_dump(object_,
path,
dump_function=dump_and_add_to_dump,
parameters=self.parameters,
to_add=to_add,
use_cpickle=self.use_cpickle)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path, ))
def do(self, callback_name, *args):
"""Pickle the main loop object to the disk.
If `*args` contain an argument from user, it is treated as
saving path to be used instead of the one given at the
construction stage.
"""
logger.info("Snapshoting has started")
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path + '_%d' % self.epoch
if from_user:
path, = from_user
to_add = None
if self.save_separately:
to_add = {
attr: getattr(self.main_loop, attr)
for attr in self.save_separately
}
if self.parameters is None:
if hasattr(self.main_loop, 'model'):
self.parameters = self.main_loop.model.parameters
object_ = None
if self.save_main_loop:
object_ = self.main_loop
secure_dump(object_,
path,
dump_function=dump_and_add_to_dump,
parameters=self.parameters,
to_add=to_add,
use_cpickle=self.use_cpickle)
self.epoch += 1
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path, ))
logger.info("Snapshoting has finished")
def do(self, callback_name, *args):
if self.notification_name in self.main_loop.log.current_row:
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
filenames = self.save_separately_filenames(path)
for attribute in self.save_separately:
secure_dump(getattr(self.main_loop, attribute),
filenames[attribute] + '.pkl',
pickle.dump,
protocol=pickle.HIGHEST_PROTOCOL)
# self.iteration += 1
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(
SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path, ))
def dump_log(self, main_loop):
secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
def dump_iteration_state(self, main_loop):
secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
def do(self, which_callback, *args, **kwargs):
secure_dump(self.main_loop, self.save_path, dump_main_loop)
def do(self, which_callback, *args):
if self.notification_name in self.main_loop.log.current_row:
secure_dump(self.main_loop, self.save_path, dump_main_loop)
def do(self, callback_name, *args):
secure_dump(self.main_loop.log, self.path, use_cpickle=True)
def dump_iteration_state(self, main_loop):
secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
def do(self, callback_name, *args):
secure_dump(self.main_loop.log, self.path, use_cpickle=True)
def dump_log(self, main_loop):
logger.info(" ...saving log")
secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
def do(self, which_callback, *args, **kwargs):
secure_dump(self.main_loop, self.save_path, dump_main_loop)
def do(self, which_callback, *args):
values = dict((variable.name, np.asarray(value))
for variable, value in zip(self.variables,
self.function(**self.batch)))
secure_dump(values, "%s_%i.pkl" % (self.save_path, self.i))
self.i += 1
def dump_log(self, main_loop):
logger.info(" ...saving log")
secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
def do(self, which_callback, *args):
values = dict((variable.name, np.asarray(value)) for variable, value in
zip(self.variables, self.function(**self.batch)))
secure_dump(values, "%s_%i.pkl" % (self.save_path, self.i))
self.i += 1
def dump_log(self, main_loop):
secure_dump(main_loop.log,
self.enhance_path(main_loop, self.path_to_log),
cPickle.dump)
def do(self, which_callback, *args):
if self.notification_name in self.main_loop.log.current_row:
secure_dump(self.main_loop, self.save_path, use_cpickle=True)
def dump_iteration_state(self, main_loop):
secure_dump(main_loop.iteration_state,
self.enhance_path(main_loop, self.path_to_iter_state))
def dump_log(self, main_loop):
secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
patience = 50
time_since_improvement = 0
best_valid_error_rate = 1
log = dict()
for epoch in range(args.num_epochs):
print "epoch", epoch
for batch in util.batches(datasets["train"], batch_size=args.batch_size):
sys.stdout.write(".")
sys.stdout.flush()
monitor = train_fn(**batch)
util.append_log(log, monitor, prefix="iteration")
print
util.print_monitor(monitor, prefix="iteration")
for which_set in "train valid test".split():
monitor = monitor_fn(**next(util.batches(datasets[which_set], batch_size=1000)))
util.append_log(log, monitor, prefix=which_set)
util.print_monitor(monitor, prefix=which_set)
serialization.secure_dump(log, "log.pkl")
if log["valid_error_rate"][-1] < best_valid_error_rate:
best_valid_error_rate = log["valid_error_rate"][-1]
serialization.secure_dump(dict((parameter.name, parameter.get_value())
for parameter in parameters.values()),
"best_parameters.pkl")
time_since_improvement = 0
else:
time_since_improvement += 1
if time_since_improvement > patience:
break