def load_npz_no_strict(filename, obj): try: serializers.load_npz(filename, obj) except KeyError as e: warnings.warn(repr(e)) with numpy.load(filename) as f: d = serializers.NpzDeserializer(f, strict=False) d.load(obj)
def load_model(filename: str) -> Tuple[Word2Vec, HyperParameters]: with np.load(filename) as f: deserializer = serializers.NpzDeserializer(f) pickled_params = deserializer("hyper_parameters", None) params = pickle.loads( pickled_params.tobytes()) # type: HyperParameters model = _make_model(params) deserializer["model"].load(model) return model, params
def test_iterator_serialize_backward_compat(self): dataset = [1, 2, 3, 4, 5, 6] it = iterators.SerialIterator(dataset, 2, shuffle=self.shuffle, order_sampler=self.order_sampler) self.assertEqual(it.epoch, 0) self.assertAlmostEqual(it.epoch_detail, 0 / 6) self.assertIsNone(it.previous_epoch_detail) batch1 = it.next() self.assertEqual(len(batch1), 2) self.assertIsInstance(batch1, list) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 2 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 0 / 6) batch2 = it.next() self.assertEqual(len(batch2), 2) self.assertIsInstance(batch2, list) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 4 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 2 / 6) target = dict() it.serialize(serializers.DictionarySerializer(target)) # older version uses '_order' target['_order'] = target['order'] del target['order'] # older version does not have previous_epoch_detail del target['previous_epoch_detail'] it = iterators.SerialIterator(dataset, 2) it.serialize(serializers.NpzDeserializer(target)) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 4 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 2 / 6) batch3 = it.next() self.assertEqual(len(batch3), 2) self.assertIsInstance(batch3, list) self.assertTrue(it.is_new_epoch) self.assertEqual(sorted(batch1 + batch2 + batch3), dataset) self.assertAlmostEqual(it.epoch_detail, 6 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 4 / 6)
def load_model(filename: str) -> EncoderDecoder: with np.load(filename) as f: d = dict(f.iteritems()) # 後方互換性のために _W を _extract_output に改名する for k, v in list(d.items()): old_prefix = "model/_W/" new_prefix = "model/_extract_output/" if k.startswith(old_prefix): d[new_prefix + k[len(old_prefix):]] = v deserializer = serializers.NpzDeserializer(d) pickled_params = deserializer("hyper_parameters", None) params = pickle.loads(pickled_params.tobytes()) model = EncoderDecoder(*params) deserializer["model"].load(model) return model
def test_iterator_compatibilty(self): dataset = [1, 2, 3, 4, 5, 6] iters = ( lambda: iterators.SerialIterator(dataset, 2), lambda: iterators.MultiprocessIterator(dataset, 2, **self.options), ) for it_before, it_after in itertools.permutations(iters, 2): it = it_before() self.assertEqual(it.epoch, 0) self.assertAlmostEqual(it.epoch_detail, 0 / 6) batch1 = it.next() self.assertEqual(len(batch1), 2) self.assertIsInstance(batch1, list) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 2 / 6) batch2 = it.next() self.assertEqual(len(batch2), 2) self.assertIsInstance(batch2, list) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 4 / 6) target = dict() it.serialize(serializers.DictionarySerializer(target)) it = it_after() it.serialize(serializers.NpzDeserializer(target)) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 4 / 6) batch3 = it.next() self.assertEqual(len(batch3), 2) self.assertIsInstance(batch3, list) self.assertTrue(it.is_new_epoch) self.assertEqual(sorted(batch1 + batch2 + batch3), dataset) self.assertAlmostEqual(it.epoch_detail, 6 / 6)
def test_iterator_serialize(self): dataset = [1, 2, 3, 4, 5, 6] it = iterators.MultiprocessIterator(dataset, 2, **self.options) self.assertEqual(it.epoch, 0) self.assertAlmostEqual(it.epoch_detail, 0 / 6) self.assertIsNone(it.previous_epoch_detail) batch1 = it.next() self.assertEqual(len(batch1), 2) self.assertIsInstance(batch1, list) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 2 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 0 / 6) batch2 = it.next() self.assertEqual(len(batch2), 2) self.assertIsInstance(batch2, list) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 4 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 2 / 6) target = dict() it.serialize(serializers.DictionarySerializer(target)) it = iterators.MultiprocessIterator(dataset, 2, **self.options) it.serialize(serializers.NpzDeserializer(target)) self.assertFalse(it.is_new_epoch) self.assertAlmostEqual(it.epoch_detail, 4 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 2 / 6) batch3 = it.next() self.assertEqual(len(batch3), 2) self.assertIsInstance(batch3, list) self.assertTrue(it.is_new_epoch) self.assertEqual(sorted(batch1 + batch2 + batch3), dataset) self.assertAlmostEqual(it.epoch_detail, 6 / 6) self.assertAlmostEqual(it.previous_epoch_detail, 4 / 6)
def load_model_flexible(filename_list, encdec): mode = "normal" if isinstance(filename_list, tuple) or isinstance(filename_list, list): if len(filename_list) == 1: filename_list = filename_list[0] else: mode = "average" if mode == "normal": log.info("loading model parameters from %s", filename_list) try: serializers.load_npz(filename_list, encdec) except KeyError: log.info("not model format, trying snapshot format") with np.load(filename_list) as fseri: dicseri = serializers.NpzDeserializer( fseri, path="updater/model:main/") dicseri.load(encdec) else: assert mode == "average" log.info("loading averaged model parameters from %r", filename_list) dseri = NpzDeserializerAverage( [np.load(filename) for filename in filename_list]) dseri.load(encdec)
def load_npz(filename, obj): with np.load(filename) as f: d = serializers.NpzDeserializer(f, strict=False) d.load(obj)
def load_from_csv(directory, model): dic_params = csv_to_npz(directory) d = serializers.NpzDeserializer(dic_params) d.load(model)
def load_checkpoint(self, save_path, epoch=-1, restart=False, load_pretrained_model=False): """Load checkpoint. Args: save_path (string): path to the saved models epoch (int, optional): if -1 means the last saved model restart (bool, optional): if True, restore the save optimizer load_pretrained_model (bool, optional): if True, load all parameters which match those of the new model's parameters Returns: epoch (int): the currnet epoch step (int): the current step lr (float): metric_dev_best (float): """ if int(epoch) == -1: # Restore the last saved model epochs = [(int(basename(x).split('-')[-1].split('.')[0]), x) for x in glob(join(save_path, 'model.*'))] if len(epochs) == 0: raise ValueError epoch = sorted(epochs, key=lambda x: x[0])[-1][0] model_path = join(save_path, 'model.epoch-' + str(epoch) + '.npz') if isfile(join(model_path)): with np.load(model_path) as f: deserializer = serializers.NpzDeserializer(f) pickled_params = deserializer("checkpoint", None) checkpoint = pickle.loads(pickled_params.tobytes()) # type: HyperParameters # Restore parameters if load_pretrained_model: logger.info( "=> Loading pre-trained checkpoint (epoch:%d): %s" % (epoch, model_path)) # TODO: # pretrained_dict = checkpoint['state_dict'] # model_dict = self.state_dict() # # # 1. filter out unnecessary keys and params which do not match size # pretrained_dict = { # k: v for k, v in pretrained_dict.items() if k in model_dict.keys() and v.size() == model_dict[k].size()} # # 2. overwrite entries in the existing state dict # model_dict.update(pretrained_dict) # # 3. load the new state dict # self.load_state_dict(model_dict) # for k in pretrained_dict.keys(): # logger.info(k) deserializer["model"].load(self) # Restore optimizer if restart: if hasattr(self, 'optimizer'): deserializer["optimizer"].load(self.optimizer) else: raise ValueError('Set optimizer.') else: print("=> Loading checkpoint (epoch:%d): %s" % (epoch, model_path)) else: raise ValueError("No checkpoint found at %s" % model_path) return (checkpoint['epoch'] + 1, checkpoint['step'] + 1, checkpoint['lr'], checkpoint['metric_dev_best'])
else: vae = ais.AIS(decoder, M=zcount, T=ais_temps, steps=ais_steps, stepsize=ais_stepsize, sigma=ais_sigma, encoder=encoder) else: sys.exit("Unsupported VAE type") #serializers.load_hdf5(model_file, vae) #serializers.load_npz(model_file, vae, path='updater/model:main/') try: with np.load(model_file) as f: d = serializers.NpzDeserializer(f, path='updater/model:main/') d.load(vae) except: with np.load(model_file) as f: d = serializers.NpzDeserializer(f, path='updater/model:elbo/') d.load(vae) print "Deserialized model '%s' of type '%s'" % (model_file, vae_type_train) if gpu_id >= 0: vae.to_gpu(gpu_id) print "Moved model to GPU %d" % gpu_id # For debugging purposes, optionally, obtain and write logw value for the # first few test samples if '--logw' in args and args['--logw'] is not None: