def test_in_memory(): skip_if_not_available(datasets=['mnist']) # Load MNIST and get two batches mnist = MNIST('train') data_stream = DataStream(mnist, iteration_scheme=SequentialScheme( num_examples=mnist.num_examples, batch_size=256)) epoch = data_stream.get_epoch_iterator() for i, (features, targets) in enumerate(epoch): if i == 1: break assert numpy.all(features == mnist.features[256:512]) # Pickle the epoch and make sure that the data wasn't dumped with tempfile.NamedTemporaryFile(delete=False) as f: filename = f.name pickle_dump(epoch, f) assert os.path.getsize(filename) < 1024 * 1024 # Less than 1MB # Reload the epoch and make sure that the state was maintained del epoch with open(filename, 'rb') as f: epoch = cPickle.load(f) features, targets = next(epoch) assert numpy.all(features == mnist.features[512:768])
def secure_pickle_dump(object_, path): """Try pickling into a temporary file and then move.""" try: dirname = os.path.dirname(path) with tempfile.NamedTemporaryFile(delete=False, dir=dirname) as temp: pickle_dump(object_, temp) shutil.move(temp.name, path) except Exception as e: # if "temp" in locals(): # os.remove(temp.name) logger.error(" Error {0}".format(str(e)))
def test_load_log(): log = TrainingLog() log[0].channel0 = 0 # test simple TrainingLog pickles with tempfile.NamedTemporaryFile() as f: pickle_dump(log, f) f.flush() log2 = plot.load_log(f.name) assert log2[0].channel0 == 0 # test MainLoop pickles main_loop = MainLoop(model=None, data_stream=None, algorithm=None, log=log) with tempfile.NamedTemporaryFile() as f: pickle_dump(main_loop, f) f.flush() log2 = plot.load_log(f.name) assert log2[0].channel0 == 0
def test_text(): # Test word level and epochs. with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: sentences1 = f.name f.write("This is a sentence\n") f.write("This another one") with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: sentences2 = f.name f.write("More sentences\n") f.write("The last one") dictionary = {'<UNK>': 0, '</S>': 1, 'this': 2, 'a': 3, 'one': 4} text_data = TextFile(files=[sentences1, sentences2], dictionary=dictionary, bos_token=None, preprocess=lower) stream = text_data.get_default_stream() epoch = stream.get_epoch_iterator() assert len(list(epoch)) == 4 epoch = stream.get_epoch_iterator() for sentence in zip(range(3), epoch): pass f = BytesIO() pickle_dump(epoch, f) sentence = next(epoch) f.seek(0) epoch = cPickle.load(f) assert next(epoch) == sentence assert_raises(StopIteration, next, epoch) # Test character level. dictionary = dict([(chr(ord('a') + i), i) for i in range(26)] + [(' ', 26)] + [('<S>', 27)] + [('</S>', 28)] + [('<UNK>', 29)]) text_data = TextFile(files=[sentences1, sentences2], dictionary=dictionary, preprocess=lower, level="character") sentence = next(text_data.get_default_stream().get_epoch_iterator())[0] assert sentence[:3] == [27, 19, 7] assert sentence[-3:] == [2, 4, 28]
def test_load_log(): log = TrainingLog() log[0].channel0 = 0 # test simple TrainingLog pickles with tempfile.NamedTemporaryFile() as f: pickle_dump(log, f) f.flush() log2 = plot.load_log(f.name) assert log2[0].channel0 == 0 # test MainLoop pickles main_loop = MainLoop(model=None, data_stream=None, algorithm=None, log=log) with tempfile.NamedTemporaryFile() as f: pickle_dump(main_loop, f) f.flush() log2 = plot.load_log(f.name) assert log2[0].channel0 == 0
def dump_log(self, main_loop): with open(self.path_to_log, "wb") as destination: pickle_dump(main_loop.log, destination)
def dump_iteration_state(self, main_loop): with open(self.path_to_iteration_state, "wb") as destination: pickle_dump(main_loop.iteration_state, destination)
def dump_log(self, main_loop): with open(self.path_to_log, "wb") as destination: pickle_dump(main_loop.log, destination)
def dump_iteration_state(self, main_loop): with open(self.path_to_iteration_state, "wb") as destination: pickle_dump(main_loop.iteration_state, destination)