def dump_iteration_state(self, main_loop): if self.slim_iteration_state: logger.info(" ...saving iteration state (slim)") secure_dump(main_loop.epoch_iterator, self.path_to_iteration_state) else: logger.info(" ...saving iteration state (full)") secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
def do(self, which_callback, *args): current_value = self.main_loop.log.current_row.get(self.record_name) if current_value is None: return best_value = self.main_loop.status.get(self.best_name, None) if (best_value is None or (current_value != best_value and self.choose_best( current_value, best_value) == current_value)): self.main_loop.status[self.best_name] = current_value # save main_loop _, from_user = self.parse_args(which_callback, args) try: path = self.path if from_user: path, = from_user secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle) filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute], use_cpickle=self.use_cpickle) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get( SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path, ))
def do(self, callback_name, *args): """Pickle the save_separately parts (and not the main loop object) to disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user ### this line is disabled from superclass impl to bypass using blocks.serialization.dump ### because pickling main thusly causes pickling error: ### "RuntimeError: maximum recursion depth exceeded while calling a Python object" # secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle) filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute], cPickle.dump) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path, ))
def do(self, callback_name, *args): """Pickle the save_separately parts (and not the main loop object) to disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user ### this line is disabled from superclass impl to bypass using blocks.serialization.dump ### because pickling main thusly causes pickling error: ### "RuntimeError: maximum recursion depth exceeded while calling a Python object" # secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle) filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute], cPickle.dump) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path,))
def do(self, callback_name, *args): """Pickle the main loop object to the disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user to_add = None if self.save_separately: to_add = {attr: getattr(self.main_loop, attr) for attr in self.save_separately} if self.parameters is None: if hasattr(self.main_loop, 'model'): self.parameters = self.main_loop.model.parameters object_ = None if self.save_main_loop: object_ = self.main_loop secure_dump(object_, path, dump_function=dump_and_add_to_dump, parameters=self.parameters, to_add=to_add, use_cpickle=self.use_cpickle) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path,))
def do(self, callback_name, *args): """Pickle the main loop object to the disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle) filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute], cPickle.dump) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path,))
def do_checkpoint(self, callback_name, *args): logger.info("Checkpointing has started") _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user to_add = None if self.save_separately: to_add = {attr: getattr(self.main_loop, attr) for attr in self.save_separately} if self.parameters is None: if hasattr(self.main_loop, 'model'): self.parameters = self.main_loop.model.parameters object_ = None if self.save_main_loop: object_ = self.main_loop secure_dump(object_, path, dump_function=dump_and_add_to_dump, parameters=self.parameters, to_add=to_add, use_cpickle=self.use_cpickle) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path,)) logger.info("Checkpointing has finished")
def _dump(self): try: self.main_loop.log.current_row['saved_best_to'] = self.path secure_dump(self.main_loop, self.path) except Exception: self.main_loop.log.current_row['saved_best_to'] = None raise
def do(self, callback_name, *args): """Pickle the main loop object to the disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user secure_dump(self.get_save(), path, use_cpickle=self.use_cpickle) filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute], cPickle.dump, protocol=DEFAULT_PROTOCOL) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path, ))
def test_secure_dump(): foo = object() bar = lambda: None # flake8: noqa with NamedTemporaryFile(delete=False, dir=config.temp_dir) as f: secure_dump(foo, f.name) assert_raises(PicklingError, secure_dump, bar, f.name) with open(f.name, 'rb') as f: assert type(load(f)) is object
def test_secure_dump(): foo = object() bar = lambda: None # flake8: noqa with NamedTemporaryFile(delete=False) as f: secure_dump(foo, f.name) assert_raises(PicklingError, secure_dump, bar, f.name) with open(f.name, 'rb') as f: assert type(load(f)) is object
def do(self, callback_name, *args): """Pickle the main loop object to the disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ sharedData = {} _, from_user = self.parse_args(callback_name, args) try: #shared data manipulation #add default values to temp share for var in self.main_loop.model.variables: if var.name != None and "sharedData" in var.name: sharedData[var.name] = var.get_value(borrow=True) if theano.config.floatX in str(var.type): self.tempIndices[theano.config.floatX] += 1 temp = self.tempShared[theano.config.floatX][ self.tempIndices[theano.config.floatX]] elif 'uint8' in str(var.type): self.tempIndices['uint8'] += 1 temp = self.tempShared['uint8'][ self.tempIndices['uint8']] else: print( 'you have not accounted for all shared variables') sys.exit(0) var.set_value(temp.get_value(borrow=True), borrow=True) path = self.path if from_user: path, = from_user secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle) filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute], cPickle.dump, protocol=DEFAULT_PROTOCOL) except Exception: path = None raise finally: self.updateIndices() already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path, )) # Change the shared variable back. for var in self.main_loop.model.variables: if var.name != None and "sharedData" in var.name and var.name in sharedData: var.set_value(sharedData[var.name], borrow=True)
def _dump(self): try: path = self.path + '/best' self.main_loop.log.current_row['saved_best_to'] = path logger.info("Saving log ...") f = open(self.path + '/log.txt', 'w') f.write(str(self.main_loop.log)) f.close() logger.info("Dumping best model ...") secure_dump(self.main_loop.model.params, path, use_cpickle=True) except Exception: self.main_loop.log.current_row['saved_best_to'] = None raise
def do(self, callback_name, *args): if self.notification_name in self.main_loop.log.current_row: _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute] + '.pkl', pickle.dump, protocol=pickle.HIGHEST_PROTOCOL) # self.iteration += 1 except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path,))
def do(self, callback_name, *args): """Pickle the main loop object to the disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ _, from_user = self.parse_args(callback_name, args) if os.path.isfile(self.path): count = int(self.path.split('.')[-1]) count += 1 self.path = ".".join(self.path.split('.')[:-1]) + "." + str(count) try: path = self.path if from_user: path, = from_user to_add = None if self.save_separately: to_add = { attr: getattr(self.main_loop, attr) for attr in self.save_separately } if self.parameters is None: if hasattr(self.main_loop, 'model'): self.parameters = self.main_loop.model.parameters object_ = None if self.save_main_loop: object_ = self.main_loop secure_dump(object_, path, dump_function=dump_and_add_to_dump, parameters=self.parameters, to_add=to_add, use_cpickle=self.use_cpickle) except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path, ))
def do(self, callback_name, *args): """Pickle the main loop object to the disk. If `*args` contain an argument from user, it is treated as saving path to be used instead of the one given at the construction stage. """ logger.info("Snapshoting has started") _, from_user = self.parse_args(callback_name, args) try: path = self.path + '_%d' % self.epoch if from_user: path, = from_user to_add = None if self.save_separately: to_add = { attr: getattr(self.main_loop, attr) for attr in self.save_separately } if self.parameters is None: if hasattr(self.main_loop, 'model'): self.parameters = self.main_loop.model.parameters object_ = None if self.save_main_loop: object_ = self.main_loop secure_dump(object_, path, dump_function=dump_and_add_to_dump, parameters=self.parameters, to_add=to_add, use_cpickle=self.use_cpickle) self.epoch += 1 except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path, )) logger.info("Snapshoting has finished")
def do(self, callback_name, *args): if self.notification_name in self.main_loop.log.current_row: _, from_user = self.parse_args(callback_name, args) try: path = self.path if from_user: path, = from_user filenames = self.save_separately_filenames(path) for attribute in self.save_separately: secure_dump(getattr(self.main_loop, attribute), filenames[attribute] + '.pkl', pickle.dump, protocol=pickle.HIGHEST_PROTOCOL) # self.iteration += 1 except Exception: path = None raise finally: already_saved_to = self.main_loop.log.current_row.get( SAVED_TO, ()) self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path, ))
def dump_log(self, main_loop): secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
def dump_iteration_state(self, main_loop): secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
def do(self, which_callback, *args, **kwargs): secure_dump(self.main_loop, self.save_path, dump_main_loop)
def do(self, which_callback, *args): if self.notification_name in self.main_loop.log.current_row: secure_dump(self.main_loop, self.save_path, dump_main_loop)
def do(self, callback_name, *args): secure_dump(self.main_loop.log, self.path, use_cpickle=True)
def dump_log(self, main_loop): logger.info(" ...saving log") secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
def do(self, which_callback, *args): values = dict((variable.name, np.asarray(value)) for variable, value in zip(self.variables, self.function(**self.batch))) secure_dump(values, "%s_%i.pkl" % (self.save_path, self.i)) self.i += 1
def dump_log(self, main_loop): secure_dump(main_loop.log, self.enhance_path(main_loop, self.path_to_log), cPickle.dump)
def do(self, which_callback, *args): if self.notification_name in self.main_loop.log.current_row: secure_dump(self.main_loop, self.save_path, use_cpickle=True)
def dump_iteration_state(self, main_loop): secure_dump(main_loop.iteration_state, self.enhance_path(main_loop, self.path_to_iter_state))
patience = 50 time_since_improvement = 0 best_valid_error_rate = 1 log = dict() for epoch in range(args.num_epochs): print "epoch", epoch for batch in util.batches(datasets["train"], batch_size=args.batch_size): sys.stdout.write(".") sys.stdout.flush() monitor = train_fn(**batch) util.append_log(log, monitor, prefix="iteration") print util.print_monitor(monitor, prefix="iteration") for which_set in "train valid test".split(): monitor = monitor_fn(**next(util.batches(datasets[which_set], batch_size=1000))) util.append_log(log, monitor, prefix=which_set) util.print_monitor(monitor, prefix=which_set) serialization.secure_dump(log, "log.pkl") if log["valid_error_rate"][-1] < best_valid_error_rate: best_valid_error_rate = log["valid_error_rate"][-1] serialization.secure_dump(dict((parameter.name, parameter.get_value()) for parameter in parameters.values()), "best_parameters.pkl") time_since_improvement = 0 else: time_since_improvement += 1 if time_since_improvement > patience: break