def test_get_section_lazy(self): """Test getting a section of the state lazily """ state['a.a'] = 12 state['a.b'] = 13 state['c'] = 24 self.assertIn('a', state) section_a = state['a'] self.assertIn('a', section_a) self.assertIn('b', section_a) self.assertNotIn('c', section_a) self.assertEqual(section_a['a'], 12) self.assertEqual(section_a['b'], 13) with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('a', state) state.load(temp.name) self.assertIn('a', state) section_a = state['a'] self.assertIn('a', section_a) self.assertIn('b', section_a) self.assertNotIn('c', section_a) self.assertEqual(section_a['a'], 12) self.assertEqual(section_a['b'], 13)
def test_persists(self): """Test if memoization persists """ called = [0] def environment(expected_calls=1): """Creates target function in scope, check number of calls """ @persistent_memoize def target(arg): """Test function...""" called[0] += 1 return 2 * arg result1 = target(1) self.assertEqual(result1, 2) self.assertEqual(called[0], expected_calls) result2 = target(1) self.assertEqual(result2, 2) self.assertEqual(called[0], expected_calls) environment(expected_calls=1) with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() environment(expected_calls=2) environment(expected_calls=2) state.reset_instance() state.load(temp.name, lazy=True) environment(expected_calls=2)
def test_load_nonexisting_noraise(self): """Test loading a state that does not exist with error flag True """ temp = tempfile.NamedTemporaryFile(delete=False) os.remove(temp.name) state.load(temp.name, lazy=False, raise_error=False) self.assertEqual(len(state), 0)
def test_lazy_really_lazy(self): """Test lazy loading is really lazy """ self._setup_basic_state() temp = tempfile.NamedTemporaryFile(delete=False) temp2 = tempfile.NamedTemporaryFile(delete=False) state.save(temp.name) # Write bogus info to state state['list'] = 'foo' state['values.int'] = 43 # This should only load the keys state.load(temp.name, lazy=True) self.assertEqual(len(state.keys()), 3) # This should raise an error shutil.move(temp.name, temp2.name) self.assertRaises(IOError, state.__getitem__, 'list') shutil.copyfile(temp2.name, temp.name) # Now it should work list_val = state['list'] self.assertEqual(self.list_val, list_val) # This should raise another error shutil.move(temp.name, temp2.name) self.assertRaises(IOError, state.__getitem__, 'values.int') shutil.copyfile(temp2.name, temp.name) # This should work again int_val = state['values.int'] self.assertEqual(self.int_val, int_val) os.remove(temp.name) os.remove(temp2.name)
def test_load_nonexist_lazy_noraise(self): """Test loading a state that does not exist with error flag default """ temp = tempfile.NamedTemporaryFile(delete=False) os.remove(temp.name) state.load(temp.name, lazy=True, raise_error=False) self.assertNotIn('foo', state)
def test_load_broken(self): """Test loading a broken state lazily """ state['a'] = 12 with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.load(temp.name) state.get_instance().filename = None self.assertRaises(RuntimeError, lambda: state['a'])
def test_in_lazy(self): """Test checking for an attribute in a lazily loaded state """ self._setup_basic_state() with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('list', state) state.load(temp.name, lazy=True) self.assertIn('list', state)
def test_saving_loading_lazy_array(self): """Test saving and loading a numpy array """ random = np.random.rand(100) state['random'] = random with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('random', state) state.load(temp.name) self.assertTrue((random == state['random']).all())
def test_triggers_state_changed(self): """Test if memoization triggers state change """ with tempfile.NamedTemporaryFile() as temp: cache = PersistentCache('bla') state.save(temp.name) state.reset_instance() state.load(temp.name, lazy=True) self.assertFalse(state.changed) cache[1] = 42 self.assertTrue(state.changed)
def test_with_block_does_save(self): """Test the with-block of the StateHandler saves if required """ state['a'] = 1 with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() with StateHandler(temp.name, save=True): state['a'] = 42 state.reset_instance() state.load(temp.name) self.assertEqual(state['a'], 42)
def test_with_block_does_not_save(self): """Test the basic with-block of the StateHandler does not save anything """ state['a'] = 1 self.assertEqual(len(state), 1) with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() with StateHandler(temp.name): state['a'] = 42 state.reset_instance() state.load(temp.name) self.assertEqual(state['a'], 1)
def test_loading_numpy_performance(self): """Test loading a numpy array and make sure it's reasonably fast """ random = np.array(np.random.randint(0, 255, 1024 * 1024)) state['random'] = random with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('random', state) tic = datetime.now() state.load(temp.name) toc = datetime.now() self.assertTrue((toc - tic).total_seconds() < 0.5)
def test_cache_persists_reset(self): """Test if the cache persists after resetting and reloading state """ cache = PersistentCache('bla') cache[1] = 42 self.assertIn(1, cache) self.assertEqual(cache[1], 42) with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() state.load(temp.name) self.assertIn(1, cache) self.assertEqual(cache[1], 42)
def test_loading_list_performance(self): """Test loading a list and make sure it's reasonably fast """ random = np.random.randint(0, 255, 1024*1024).tolist() state['random'] = random with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('random', state) tic = datetime.now() state.load(temp.name) toc = datetime.now() self.assertTrue((toc - tic).total_seconds() < 0.5)
def test_get_section_lazy2(self): """Test getting directly a section of the state lazily """ state['a.b'] = 12 self.assertIn('a.b', state) self.assertEqual(state['a.b'], 12) with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() state.load(temp.name) self.assertIn('a.b', state) self.assertEqual(state['a.b'], 12)
def test_no_unnecessary_save_lazy(self): """Test saving the state does not save just after loading lazily """ self.assertFalse(state.need_saving()) state['bla'] = 'bla' self.assertTrue(state.need_saving()) with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) self.assertFalse(state.need_saving()) state.reset_instance() self.assertFalse(state.need_saving()) state.load(temp.name, lazy=True) self.assertFalse(state.need_saving()) self.assertEqual(state['bla'], 'bla') self.assertFalse(state.need_saving())
def test_save_load_file_keys(self): """Test saving file and reloading yields identical keys """ state['normal_key'] = 1 state['dashed-key'] = 3 state['dotted.key'] = 4 with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() state.load(temp.name, lazy=False) # Make sure the keys are there for key in ['normal_key', 'dashed-key', 'dotted.key']: self.assertIn(key, state)
def test_saving_unloaded_value(self): """Test saving does not delete unloaded values """ state['a'] = 12 with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('a', state) state.load(temp.name) state['b'] = 12 state.save(temp.name) state.reset_instance() self.assertNotIn('a', state) self.assertNotIn('b', state) state.load(temp.name) self.assertIn('a', state) self.assertIn('b', state)
def test_saving_deleted_value(self): """Test saving really deletes entry """ state['a'] = 12 with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('a', state) state.load(temp.name) self.assertIn('a', state) del state['a'] self.assertNotIn('a', state) state.save(temp.name) state.reset_instance() self.assertNotIn('a', state) state.load(temp.name) self.assertNotIn('a', state)
def show_state(*arguments): """Shows the contents of the state loaded by the configuration or from the file specified as an argument. """ if len(arguments) == 0: state_file = conf['pyexperiment.state_filename'] else: state_file = arguments[0] print_bold("Load state from file '%s'", state_file) try: state.load(state_file, lazy=False, raise_error=True) except IOError as err: print(err) else: if len(state) > 0: state.show() else: print("State empty")
def test_saving_loading_np_array2(self): """Test saving and loading numpy arrays of higher dimension """ random1 = np.random.rand(321, 123) random2 = np.random.randint(0, 100, (123, 345)) state['random1'] = random1 state['random2'] = random2 with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state.reset_instance() self.assertNotIn('random1', state) self.assertNotIn('random2', state) state.load(temp.name, lazy=False) self.assertTrue((random1 == state['random1']).all()) self.assertTrue((random2 == state['random2']).all()) self.assertEqual(state['random1'].shape, random1.shape) self.assertEqual(state['random2'].shape, random2.shape) self.assertEqual(state['random1'].dtype, random1.dtype) self.assertEqual(state['random2'].dtype, random2.dtype)
def test_save_load_file_lazy(self): """Test saving file and reloading lazily yields identical values """ self._setup_basic_state() with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) # Write bogus info to state state['list'] = 'foo' state['dict'] = 'bar' state['values.int'] = 43 state.load(temp.name, lazy=True) list_val = state['list'] dict_val = state['dict'] int_val = state['values.int'] self.assertEqual(self.list_val, list_val) self.assertEqual(self.dict_val, dict_val) self.assertEqual(self.int_val, int_val)
def test_show_lazy(self): """Test showing the state lazily loaded """ state['a.b'] = 12 buf = io.StringIO() with tempfile.NamedTemporaryFile() as temp: state.save(temp.name) state['a.b'] = 13 state.load(temp.name, lazy=True) with stdout_redirector(buf): state.show() self.assertNotEqual(len(buf.getvalue()), 0) self.assertRegexpMatches(buf.getvalue(), r"\[a\]") self.assertRegexpMatches(buf.getvalue(), r"b") self.assertRegexpMatches(buf.getvalue(), r"12") if six.PY2: self.assertNotRegexpMatches(buf.getvalue(), r"13") elif six.PY3: self.assertNotRegex( # pylint: disable=E1101 buf.getvalue(), r"13") else: raise RuntimeError("Python version not supported")
def test_save_rollover(self): """Test saving file with rollover """ # Write some stuff to the state state['a'] = (-1)**2 state['b'] = (-1)**3 state['c'] = 41 with tempfile.NamedTemporaryFile() as temp: # Save original state state.save(temp.name, rotate_n_state_files=2) for i in range(10): # Write bogus info to state state['a'] = i**2 state['b'] = i**3 state['c'] = 42 + i state.save(temp.name, rotate_n_state_files=2) # Load last file and check contents state.load(temp.name + ".1", lazy=True) self.assertEqual(state['a'], (i - 1)**2) self.assertEqual(state['b'], (i - 1)**3) self.assertEqual(state['c'], 42 + (i - 1)) if i > 0: # Load previous to last file and check contents state.load(temp.name + ".2", lazy=True) self.assertEqual(state['a'], (i - 2)**2) self.assertEqual(state['b'], (i - 2)**3) self.assertEqual(state['c'], 42 + (i - 2)) # Load current state and check contents state.load(temp.name, lazy=True) self.assertEqual(state['a'], i**2) self.assertEqual(state['b'], i**3) self.assertEqual(state['c'], 42 + i) # Remove temp files os.remove(temp.name + ".1") os.remove(temp.name + ".2")
def test_save_rollover(self): """Test saving file with rollover """ # Write some stuff to the state state['a'] = (-1) ** 2 state['b'] = (-1) ** 3 state['c'] = 41 with tempfile.NamedTemporaryFile() as temp: # Save original state state.save(temp.name, rotate_n_state_files=2) for i in range(10): # Write bogus info to state state['a'] = i ** 2 state['b'] = i ** 3 state['c'] = 42 + i state.save(temp.name, rotate_n_state_files=2) # Load last file and check contents state.load(temp.name + ".1", lazy=True) self.assertEqual(state['a'], (i - 1) ** 2) self.assertEqual(state['b'], (i - 1) ** 3) self.assertEqual(state['c'], 42 + (i - 1)) if i > 0: # Load previous to last file and check contents state.load(temp.name + ".2", lazy=True) self.assertEqual(state['a'], (i - 2) ** 2) self.assertEqual(state['b'], (i - 2) ** 3) self.assertEqual(state['c'], 42 + (i - 2)) # Load current state and check contents state.load(temp.name, lazy=True) self.assertEqual(state['a'], i ** 2) self.assertEqual(state['b'], i ** 3) self.assertEqual(state['c'], 42 + i) # Remove temp files os.remove(temp.name + ".1") os.remove(temp.name + ".2")
def test_save_rollover_different(self): """Test saving file with one-file rollover and a different filename """ # Write some stuff to the state state['a'] = (-1) ** 2 state['b'] = (-1) ** 3 state['c'] = 41 with tempfile.NamedTemporaryFile() as temp, \ tempfile.NamedTemporaryFile() as temp2: # Save original state state.save(temp2.name, rotate_n_state_files=1) state.load(temp2.name) state['a'] = (-1) ** 2 state.save(temp.name, rotate_n_state_files=1) for i in range(10): # Write bogus info to state state['a'] = i ** 2 state['b'] = i ** 3 state['c'] = 42 + i state.save(temp.name, rotate_n_state_files=1) # Load last file and check contents state.load(temp.name + ".1", lazy=True) self.assertEqual(state['a'], (i - 1) ** 2) self.assertEqual(state['b'], (i - 1) ** 3) self.assertEqual(state['c'], 42 + (i - 1)) # Load current state and check contents state.load(temp.name, lazy=True) self.assertEqual(state['a'], i ** 2) self.assertEqual(state['b'], i ** 3) self.assertEqual(state['c'], 42 + i) # Remove temp files os.remove(temp.name + ".1") os.remove(temp2.name + ".1")
def test_save_rollover_different(self): """Test saving file with one-file rollover and a different filename """ # Write some stuff to the state state['a'] = (-1)**2 state['b'] = (-1)**3 state['c'] = 41 with tempfile.NamedTemporaryFile() as temp, \ tempfile.NamedTemporaryFile() as temp2: # Save original state state.save(temp2.name, rotate_n_state_files=1) state.load(temp2.name) state['a'] = (-1)**2 state.save(temp.name, rotate_n_state_files=1) for i in range(10): # Write bogus info to state state['a'] = i**2 state['b'] = i**3 state['c'] = 42 + i state.save(temp.name, rotate_n_state_files=1) # Load last file and check contents state.load(temp.name + ".1", lazy=True) self.assertEqual(state['a'], (i - 1)**2) self.assertEqual(state['b'], (i - 1)**3) self.assertEqual(state['c'], 42 + (i - 1)) # Load current state and check contents state.load(temp.name, lazy=True) self.assertEqual(state['a'], i**2) self.assertEqual(state['b'], i**3) self.assertEqual(state['c'], 42 + i) # Remove temp files os.remove(temp.name + ".1") os.remove(temp2.name + ".1")