Esempio n. 1
0
 def dump_iteration_state(self, main_loop):
     if self.slim_iteration_state:
         logger.info(" ...saving iteration state (slim)")
         secure_dump(main_loop.epoch_iterator, self.path_to_iteration_state)
     else:
         logger.info(" ...saving iteration state (full)")
         secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
Esempio n. 2
0
 def do(self, which_callback, *args):
     current_value = self.main_loop.log.current_row.get(self.record_name)
     if current_value is None:
         return
     best_value = self.main_loop.status.get(self.best_name, None)
     if (best_value is None
             or (current_value != best_value and self.choose_best(
                 current_value, best_value) == current_value)):
         self.main_loop.status[self.best_name] = current_value
         # save main_loop
         _, from_user = self.parse_args(which_callback, args)
         try:
             path = self.path
             if from_user:
                 path, = from_user
             secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
             filenames = self.save_separately_filenames(path)
             for attribute in self.save_separately:
                 secure_dump(getattr(self.main_loop, attribute),
                             filenames[attribute],
                             use_cpickle=self.use_cpickle)
         except Exception:
             path = None
             raise
         finally:
             already_saved_to = self.main_loop.log.current_row.get(
                 SAVED_TO, ())
             self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                         (path, ))
Esempio n. 3
0
    def do(self, callback_name, *args):
        """Pickle the save_separately parts (and not the main loop object) to disk.

        If `*args` contain an argument from user, it is treated as
        saving path to be used instead of the one given at the
        construction stage.

        """
        _, from_user = self.parse_args(callback_name, args)
        try:
            path = self.path
            if from_user:
                path, = from_user
            ### this line is disabled from superclass impl to bypass using blocks.serialization.dump
            ### because pickling main thusly causes pickling error:
            ### "RuntimeError: maximum recursion depth exceeded while calling a Python object"
            # secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
            filenames = self.save_separately_filenames(path)
            for attribute in self.save_separately:
                secure_dump(getattr(self.main_loop, attribute),
                            filenames[attribute], cPickle.dump)
        except Exception:
            path = None
            raise
        finally:
            already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
            self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                        (path, ))
Esempio n. 4
0
    def do(self, callback_name, *args):
        """Pickle the save_separately parts (and not the main loop object) to disk.

        If `*args` contain an argument from user, it is treated as
        saving path to be used instead of the one given at the
        construction stage.

        """
        _, from_user = self.parse_args(callback_name, args)
        try:
            path = self.path
            if from_user:
                path, = from_user
            ### this line is disabled from superclass impl to bypass using blocks.serialization.dump
            ### because pickling main thusly causes pickling error:
            ### "RuntimeError: maximum recursion depth exceeded while calling a Python object"
            # secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
            filenames = self.save_separately_filenames(path)
            for attribute in self.save_separately:
                secure_dump(getattr(self.main_loop, attribute),
                            filenames[attribute], cPickle.dump)
        except Exception:
            path = None
            raise
        finally:
            already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
            self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                        (path,))
Esempio n. 5
0
    def do(self, callback_name, *args):
        """Pickle the main loop object to the disk.

        If `*args` contain an argument from user, it is treated as
        saving path to be used instead of the one given at the
        construction stage.

        """
        _, from_user = self.parse_args(callback_name, args)
        try:
            path = self.path
            if from_user:
                path, = from_user
            to_add = None
            if self.save_separately:
                to_add = {attr: getattr(self.main_loop, attr) for attr in
                          self.save_separately}
            if self.parameters is None:
                if hasattr(self.main_loop, 'model'):
                    self.parameters = self.main_loop.model.parameters
            object_ = None
            if self.save_main_loop:
                object_ = self.main_loop
            secure_dump(object_, path,
                        dump_function=dump_and_add_to_dump,
                        parameters=self.parameters,
                        to_add=to_add,
                        use_cpickle=self.use_cpickle)
        except Exception:
            path = None
            raise
        finally:
            already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
            self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                        (path,))
Esempio n. 6
0
    def do(self, callback_name, *args):
        """Pickle the main loop object to the disk.

        If `*args` contain an argument from user, it is treated as
        saving path to be used instead of the one given at the
        construction stage.

        """
        _, from_user = self.parse_args(callback_name, args)
        try:
            path = self.path
            if from_user:
                path, = from_user
            secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
            filenames = self.save_separately_filenames(path)
            for attribute in self.save_separately:
                secure_dump(getattr(self.main_loop, attribute),
                            filenames[attribute], cPickle.dump)
        except Exception:
            path = None
            raise
        finally:
            already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
            self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                        (path,))
 def do_checkpoint(self, callback_name, *args):
     logger.info("Checkpointing has started")
     _, from_user = self.parse_args(callback_name, args)
     try:
         path = self.path
         if from_user:
             path, = from_user
         to_add = None
         if self.save_separately:
             to_add = {attr: getattr(self.main_loop, attr) for attr in
                       self.save_separately}
         if self.parameters is None:
             if hasattr(self.main_loop, 'model'):
                 self.parameters = self.main_loop.model.parameters
         object_ = None
         if self.save_main_loop:
             object_ = self.main_loop
         secure_dump(object_, path,
                     dump_function=dump_and_add_to_dump,
                     parameters=self.parameters,
                     to_add=to_add,
                     use_cpickle=self.use_cpickle)
     except Exception:
         path = None
         raise
     finally:
         already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
         self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                     (path,))
         logger.info("Checkpointing has finished")
Esempio n. 8
0
 def _dump(self):
     try:
         self.main_loop.log.current_row['saved_best_to'] = self.path
         secure_dump(self.main_loop, self.path)
     except Exception:
         self.main_loop.log.current_row['saved_best_to'] = None
         raise
 def _dump(self):
     try:
         self.main_loop.log.current_row['saved_best_to'] = self.path
         secure_dump(self.main_loop, self.path)
     except Exception:
         self.main_loop.log.current_row['saved_best_to'] = None
         raise
Esempio n. 10
0
    def do(self, callback_name, *args):
        """Pickle the main loop object to the disk.

        If `*args` contain an argument from user, it is treated as
        saving path to be used instead of the one given at the
        construction stage.

        """
        _, from_user = self.parse_args(callback_name, args)
        try:
            path = self.path
            if from_user:
                path, = from_user
            secure_dump(self.get_save(), path, use_cpickle=self.use_cpickle)
            filenames = self.save_separately_filenames(path)
            for attribute in self.save_separately:
                secure_dump(getattr(self.main_loop, attribute),
                            filenames[attribute],
                            cPickle.dump,
                            protocol=DEFAULT_PROTOCOL)
        except Exception:
            path = None
            raise
        finally:
            already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
            self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                        (path, ))
Esempio n. 11
0
def test_secure_dump():
    foo = object()
    bar = lambda: None  # flake8: noqa
    with NamedTemporaryFile(delete=False, dir=config.temp_dir) as f:
        secure_dump(foo, f.name)
    assert_raises(PicklingError, secure_dump, bar, f.name)
    with open(f.name, 'rb') as f:
        assert type(load(f)) is object
Esempio n. 12
0
def test_secure_dump():
    foo = object()
    bar = lambda: None  # flake8: noqa
    with NamedTemporaryFile(delete=False) as f:
        secure_dump(foo, f.name)
    assert_raises(PicklingError, secure_dump, bar, f.name)
    with open(f.name, 'rb') as f:
        assert type(load(f)) is object
Esempio n. 13
0
 def dump_iteration_state(self, main_loop):
     if self.slim_iteration_state:
         logger.info(" ...saving iteration state (slim)")
         secure_dump(main_loop.epoch_iterator, self.path_to_iteration_state)
     else:
         logger.info(" ...saving iteration state (full)")
         secure_dump(main_loop.iteration_state,
                     self.path_to_iteration_state)
Esempio n. 14
0
    def do(self, callback_name, *args):
        """Pickle the main loop object to the disk.
        If `*args` contain an argument from user, it is treated as
        saving path to be used instead of the one given at the
        construction stage.
        """
        sharedData = {}
        _, from_user = self.parse_args(callback_name, args)
        try:

            #shared data manipulation

            #add default values to temp share
            for var in self.main_loop.model.variables:
                if var.name != None and "sharedData" in var.name:
                    sharedData[var.name] = var.get_value(borrow=True)
                    if theano.config.floatX in str(var.type):
                        self.tempIndices[theano.config.floatX] += 1
                        temp = self.tempShared[theano.config.floatX][
                            self.tempIndices[theano.config.floatX]]
                    elif 'uint8' in str(var.type):
                        self.tempIndices['uint8'] += 1
                        temp = self.tempShared['uint8'][
                            self.tempIndices['uint8']]
                    else:
                        print(
                            'you have not accounted for all shared variables')
                        sys.exit(0)
                    var.set_value(temp.get_value(borrow=True), borrow=True)

            path = self.path
            if from_user:
                path, = from_user
            secure_dump(self.main_loop, path, use_cpickle=self.use_cpickle)
            filenames = self.save_separately_filenames(path)
            for attribute in self.save_separately:
                secure_dump(getattr(self.main_loop, attribute),
                            filenames[attribute],
                            cPickle.dump,
                            protocol=DEFAULT_PROTOCOL)
        except Exception:
            path = None
            raise
        finally:
            self.updateIndices()
            already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
            self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                        (path, ))
            # Change the shared variable back.
            for var in self.main_loop.model.variables:
                if var.name != None and "sharedData" in var.name and var.name in sharedData:
                    var.set_value(sharedData[var.name], borrow=True)
Esempio n. 15
0
 def _dump(self):
     try:
         path = self.path + '/best'
         self.main_loop.log.current_row['saved_best_to'] = path
         logger.info("Saving log ...")
         f = open(self.path + '/log.txt', 'w')
         f.write(str(self.main_loop.log))
         f.close()
         logger.info("Dumping best model ...")
         secure_dump(self.main_loop.model.params, path, use_cpickle=True)
     except Exception:
         self.main_loop.log.current_row['saved_best_to'] = None
         raise
Esempio n. 16
0
 def _dump(self):
     try:
         path = self.path + '/best'
         self.main_loop.log.current_row['saved_best_to'] = path
         logger.info("Saving log ...")
         f = open(self.path + '/log.txt', 'w')
         f.write(str(self.main_loop.log))
         f.close()
         logger.info("Dumping best model ...")
         secure_dump(self.main_loop.model.params, path, use_cpickle=True)
     except Exception:
         self.main_loop.log.current_row['saved_best_to'] = None
         raise
Esempio n. 17
0
 def do(self, callback_name, *args):
     if self.notification_name in self.main_loop.log.current_row:
         _, from_user = self.parse_args(callback_name, args)
         try:
             path = self.path
             if from_user:
                 path, = from_user
             filenames = self.save_separately_filenames(path)
             for attribute in self.save_separately:
                 secure_dump(getattr(self.main_loop, attribute), filenames[attribute] + '.pkl', pickle.dump, protocol=pickle.HIGHEST_PROTOCOL)
                 # self.iteration += 1
         except Exception:
             path = None
             raise
         finally:
             already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
             self.main_loop.log.current_row[SAVED_TO] = (already_saved_to + (path,))
Esempio n. 18
0
 def do(self, callback_name, *args):
     """Pickle the main loop object to the disk.
     If `*args` contain an argument from user, it is treated as
     saving path to be used instead of the one given at the
     construction stage.
     """
     _, from_user = self.parse_args(callback_name, args)
     if os.path.isfile(self.path):
         count = int(self.path.split('.')[-1])
         count += 1
         self.path = ".".join(self.path.split('.')[:-1]) + "." + str(count)
     try:
         path = self.path
         if from_user:
             path, = from_user
         to_add = None
         if self.save_separately:
             to_add = {
                 attr: getattr(self.main_loop, attr)
                 for attr in self.save_separately
             }
         if self.parameters is None:
             if hasattr(self.main_loop, 'model'):
                 self.parameters = self.main_loop.model.parameters
         object_ = None
         if self.save_main_loop:
             object_ = self.main_loop
         secure_dump(object_,
                     path,
                     dump_function=dump_and_add_to_dump,
                     parameters=self.parameters,
                     to_add=to_add,
                     use_cpickle=self.use_cpickle)
     except Exception:
         path = None
         raise
     finally:
         already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
         self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                     (path, ))
Esempio n. 19
0
 def do(self, callback_name, *args):
     """Pickle the main loop object to the disk.
     If `*args` contain an argument from user, it is treated as
     saving path to be used instead of the one given at the
     construction stage.
     """
     logger.info("Snapshoting has started")
     _, from_user = self.parse_args(callback_name, args)
     try:
         path = self.path + '_%d' % self.epoch
         if from_user:
             path, = from_user
         to_add = None
         if self.save_separately:
             to_add = {
                 attr: getattr(self.main_loop, attr)
                 for attr in self.save_separately
             }
         if self.parameters is None:
             if hasattr(self.main_loop, 'model'):
                 self.parameters = self.main_loop.model.parameters
         object_ = None
         if self.save_main_loop:
             object_ = self.main_loop
         secure_dump(object_,
                     path,
                     dump_function=dump_and_add_to_dump,
                     parameters=self.parameters,
                     to_add=to_add,
                     use_cpickle=self.use_cpickle)
         self.epoch += 1
     except Exception:
         path = None
         raise
     finally:
         already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
         self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                     (path, ))
         logger.info("Snapshoting has finished")
Esempio n. 20
0
 def do(self, callback_name, *args):
     if self.notification_name in self.main_loop.log.current_row:
         _, from_user = self.parse_args(callback_name, args)
         try:
             path = self.path
             if from_user:
                 path, = from_user
             filenames = self.save_separately_filenames(path)
             for attribute in self.save_separately:
                 secure_dump(getattr(self.main_loop, attribute),
                             filenames[attribute] + '.pkl',
                             pickle.dump,
                             protocol=pickle.HIGHEST_PROTOCOL)
                 # self.iteration += 1
         except Exception:
             path = None
             raise
         finally:
             already_saved_to = self.main_loop.log.current_row.get(
                 SAVED_TO, ())
             self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
                                                         (path, ))
Esempio n. 21
0
 def dump_log(self, main_loop):
     secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
Esempio n. 22
0
 def dump_iteration_state(self, main_loop):
     secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
Esempio n. 23
0
 def do(self, which_callback, *args, **kwargs):
     secure_dump(self.main_loop, self.save_path, dump_main_loop)
Esempio n. 24
0
 def do(self, which_callback, *args):
     if self.notification_name in self.main_loop.log.current_row:
         secure_dump(self.main_loop, self.save_path, dump_main_loop)
Esempio n. 25
0
 def do(self, callback_name, *args):
     secure_dump(self.main_loop.log, self.path, use_cpickle=True)
Esempio n. 26
0
 def dump_iteration_state(self, main_loop):
     secure_dump(main_loop.iteration_state, self.path_to_iteration_state)
Esempio n. 27
0
 def do(self, callback_name, *args):
     secure_dump(self.main_loop.log, self.path, use_cpickle=True)
Esempio n. 28
0
 def dump_log(self, main_loop):
     logger.info(" ...saving log")
     secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
Esempio n. 29
0
 def do(self, which_callback, *args, **kwargs):
     secure_dump(self.main_loop, self.save_path, dump_main_loop)
Esempio n. 30
0
 def do(self, which_callback, *args):
     values = dict((variable.name, np.asarray(value))
                   for variable, value in zip(self.variables,
                                              self.function(**self.batch)))
     secure_dump(values, "%s_%i.pkl" % (self.save_path, self.i))
     self.i += 1
Esempio n. 31
0
 def dump_log(self, main_loop):
     logger.info(" ...saving log")
     secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
Esempio n. 32
0
 def do(self, which_callback, *args):
     values = dict((variable.name, np.asarray(value)) for variable, value in
                   zip(self.variables, self.function(**self.batch)))
     secure_dump(values, "%s_%i.pkl" % (self.save_path, self.i))
     self.i += 1
Esempio n. 33
0
 def dump_log(self, main_loop):
     secure_dump(main_loop.log,
                 self.enhance_path(main_loop, self.path_to_log),
                 cPickle.dump)
Esempio n. 34
0
 def do(self, which_callback, *args):
     if self.notification_name in self.main_loop.log.current_row:
         secure_dump(self.main_loop, self.save_path, use_cpickle=True)
Esempio n. 35
0
 def dump_iteration_state(self, main_loop):
     secure_dump(main_loop.iteration_state,
                 self.enhance_path(main_loop, self.path_to_iter_state))
Esempio n. 36
0
 def dump_log(self, main_loop):
     secure_dump(main_loop.log, self.path_to_log, cPickle.dump)
Esempio n. 37
0
    patience = 50
    time_since_improvement = 0
    best_valid_error_rate = 1

    log = dict()
    for epoch in range(args.num_epochs):
        print "epoch", epoch
        for batch in util.batches(datasets["train"], batch_size=args.batch_size):
            sys.stdout.write(".")
            sys.stdout.flush()
            monitor = train_fn(**batch)
            util.append_log(log, monitor, prefix="iteration")
        print
        util.print_monitor(monitor, prefix="iteration")
        for which_set in "train valid test".split():
            monitor = monitor_fn(**next(util.batches(datasets[which_set], batch_size=1000)))
            util.append_log(log, monitor, prefix=which_set)
            util.print_monitor(monitor, prefix=which_set)
        serialization.secure_dump(log, "log.pkl")

        if log["valid_error_rate"][-1] < best_valid_error_rate:
            best_valid_error_rate = log["valid_error_rate"][-1]
            serialization.secure_dump(dict((parameter.name, parameter.get_value())
                                           for parameter in parameters.values()),
                                      "best_parameters.pkl")
            time_since_improvement = 0
        else:
            time_since_improvement += 1
        if time_since_improvement > patience:
            break